TASK: Add changes from customer project

Classes/Connection/Elasticsearch/IndexFactory.php

@@ -39,6 +39,21 @@ class IndexFactory implements Singleton
      */
     protected $configuration;
 
+    /**
+     * @var \TYPO3\CMS\Core\Log\Logger
+     */
+    protected $logger;
+
+    /**
+     * Inject log manager to get concrete logger from it.
+     *
+     * @param \TYPO3\CMS\Core\Log\LogManager $logManager
+     */
+    public function injectLogger(\TYPO3\CMS\Core\Log\LogManager $logManager)
+    {
+        $this->logger = $logManager->getLogger(__CLASS__);
+    }
+
     /**
      * @param ConfigurationContainerInterface $configuration
      */
@@ -60,7 +75,10 @@ class IndexFactory implements Singleton
         $index = $connection->getClient()->getIndex('typo3content');
 
         if ($index->exists() === false) {
-            $index->create($this->getConfigurationFor($documentType));
+            $config = $this->getConfigurationFor($documentType);
+            $this->logger->debug(sprintf('Create index %s.', $documentType), [$documentType, $config]);
+            $index->create($config);
+            $this->logger->debug(sprintf('Created index %s.', $documentType), [$documentType]);
         }
 
         return $index;
@@ -76,9 +94,11 @@ class IndexFactory implements Singleton
         try {
             $configuration = $this->configuration->get('indexing.' . $documentType . '.index');
 
-            if (isset($configuration['analysis']['analyzer'])) {
+            foreach (['analyzer', 'filter'] as $optionsToExpand) {
-                foreach ($configuration['analysis']['analyzer'] as $key => $analyzer) {
+                if (isset($configuration['analysis'][$optionsToExpand])) {
-                    $configuration['analysis']['analyzer'][$key] = $this->prepareAnalyzerConfiguration($analyzer);
+                    foreach ($configuration['analysis'][$optionsToExpand] as $key => $options) {
+                        $configuration['analysis'][$optionsToExpand][$key] = $this->prepareOptions($options);
+                    }
                 }
             }
 
@@ -89,20 +109,20 @@ class IndexFactory implements Singleton
     }
 
     /**
-     * @param array $analyzer
+     * @param array $options
      *
      * @return array
      */
-    protected function prepareAnalyzerConfiguration(array $analyzer)
+    protected function prepareOptions(array $options)
     {
-        $fieldsToExplode = ['char_filter', 'filter'];
+        $fieldsToExplode = ['char_filter', 'filter', 'word_list'];
 
         foreach ($fieldsToExplode as $fieldToExplode) {
-            if (isset($analyzer[$fieldToExplode])) {
+            if (isset($options[$fieldToExplode])) {
-                $analyzer[$fieldToExplode] = GeneralUtility::trimExplode(',', $analyzer[$fieldToExplode], true);
+                $options[$fieldToExplode] = GeneralUtility::trimExplode(',', $options[$fieldToExplode], true);
             }
         }
 
-        return $analyzer;
+        return $options;
     }
 }

Classes/Connection/Elasticsearch/SearchResult.php

@@ -83,7 +83,7 @@ class SearchResult implements SearchResultInterface
     /**
      * Return all facets, if any.
      *
-     * @return array<FacetIterface>
+     * @return array<FacetInterface>
      */
     public function getFacets()
     {

Classes/Connection/SearchResultInterface.php

@@ -35,7 +35,7 @@ interface SearchResultInterface extends \Iterator, \Countable, QueryResultInterf
     /**
      * Return all facets, if any.
      *
-     * @return array<FacetIterface>
+     * @return array<FacetInterface>
      */
     public function getFacets();
 

Classes/DataProcessing/CopyToProcessor.php

@@ -0,0 +1,54 @@
+<?php
+namespace Codappix\SearchCore\DataProcessing;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * Copies values from one field to another one.
+ */
+class CopyToProcessor implements ProcessorInterface
+{
+    public function processRecord(array $record, array $configuration)
+    {
+        $all = [];
+
+        $this->addArray($all, $record);
+        $all = array_filter($all);
+        $record[$configuration['to']] = implode(PHP_EOL, $all);
+
+        return $record;
+    }
+
+    /**
+     * @param array &$to
+     * @param array $from
+     */
+    protected function addArray(array &$to, array $from)
+    {
+        foreach ($from as $property => $value) {
+            if (is_array($value)) {
+                $this->addArray($to, $value, $exclude);
+                continue;
+            }
+
+            $to[] = (string) $value;
+        }
+    }
+}

Classes/DataProcessing/ProcessorInterface.php

@@ -0,0 +1,39 @@
+<?php
+namespace Codappix\SearchCore\DataProcessing;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * All DataProcessing Processor should implement this interface, otherwise they
+ * will not be executed.
+ */
+interface ProcessorInterface
+{
+    /**
+     * Processes the given record.
+     * Also retrieves the configuration for this processor instance.
+     *
+     * @param array $record
+     * @param array $configuration
+     *
+     * @return array
+     */
+    public function processRecord(array $record, array $configuration);
+}

Classes/Domain/Index/TcaIndexer/PagesIndexer.php

@@ -34,6 +34,12 @@ class PagesIndexer extends TcaIndexer
      */
     protected $contentTableService;
 
+    /**
+     * @var \TYPO3\CMS\Core\Resource\FileRepository
+     * @inject
+     */
+    protected $fileRepository;
+
     /**
      * @param TcaTableService $tcaTableService
      * @param TcaTableService $tcaTableService
@@ -65,7 +71,9 @@ class PagesIndexer extends TcaIndexer
             }
         }
 
-        $record['content'] = $this->fetchContentForPage($record['uid']);
+        $content = $this->fetchContentForPage($record['uid']);
+        $record['content'] = $content['content'];
+        $record['media'] = array_unique(array_merge($record['media'], $content['images']));
         parent::prepareRecord($record);
     }
 
@@ -88,14 +96,42 @@ class PagesIndexer extends TcaIndexer
         }
 
         $this->logger->debug('Fetched content for page ' . $uid);
+        $images = [];
         $content = [];
         foreach ($contentElements as $contentElement) {
+            $images = array_merge(
+                $images,
+                $this->getContentElementImages($contentElement['uid'])
+            );
             $content[] = $contentElement['bodytext'];
         }
 
+        return [
             // Remove Tags.
             // Interpret escaped new lines and special chars.
             // Trim, e.g. trailing or leading new lines.
-        return trim(stripcslashes(strip_tags(implode(' ', $content))));
+            'content' => trim(stripcslashes(strip_tags(implode(' ', $content)))),
+            'images' => $images,
+        ];
+    }
+
+    /**
+     * @param int $uidOfContentElement
+     * @return array
+     */
+    protected function getContentElementImages($uidOfContentElement)
+    {
+        $imageRelationUids = [];
+        $imageRelations = $this->fileRepository->findByRelation(
+            'tt_content',
+            'image',
+            $uidOfContentElement
+        );
+
+        foreach ($imageRelations as $relation) {
+            $imageRelationUids[] = $relation->getUid();
+        }
+
+        return $imageRelationUids;
     }
 }

Classes/Domain/Index/TcaIndexer/RelationResolver.php

@@ -86,6 +86,9 @@ class RelationResolver implements Singleton
      */
     protected function resolveValue($value, array $config)
     {
+        if (isset($config['foreign_table']) && $config['foreign_table'] === 'sys_file_reference') {
+            return $this->resolveFalRelations($value);
+        }
         if ($value === '' || $value === '0') {
             return '';
         }
@@ -102,6 +105,17 @@ class RelationResolver implements Singleton
         return '';
     }
 
+    /**
+     * @param string $value
+     * @return array
+     */
+    protected function resolveFalRelations($value)
+    {
+        $files = GeneralUtility::trimExplode(',', $value);
+        $files = array_filter($files);
+        return array_map('intval', $files);
+    }
+
     /**
      * @param array Column config.
      * @return bool

Classes/Domain/Index/TcaIndexer/TcaTableService.php

@@ -21,6 +21,8 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer;
  */
 
 use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
+use Codappix\SearchCore\Configuration\InvalidArgumentException as InvalidConfigurationArgumentException;
+use Codappix\SearchCore\DataProcessing\ProcessorInterface;
 use Codappix\SearchCore\Domain\Index\IndexingException;
 use TYPO3\CMS\Backend\Utility\BackendUtility;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
@@ -148,6 +150,17 @@ class TcaTableService
     {
         $this->relationResolver->resolveRelationsForRecord($this, $record);
 
+        try {
+            foreach ($this->configuration->get('indexing.' . $this->tableName . '.dataProcessing') as $configuration) {
+                $dataProcessor = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance($configuration['_typoScriptNodeValue']);
+                if ($dataProcessor instanceof ProcessorInterface) {
+                    $record = $dataProcessor->processRecord($record, $configuration);
+                }
+            }
+        } catch (InvalidConfigurationArgumentException $e) {
+            // Nothing to do.
+        }
+
         if (isset($record['uid']) && !isset($record['search_identifier'])) {
             $record['search_identifier'] = $record['uid'];
         }

Classes/Domain/Search/QueryFactory.php

@@ -25,6 +25,7 @@ use Codappix\SearchCore\Configuration\InvalidArgumentException;
 use Codappix\SearchCore\Connection\ConnectionInterface;
 use Codappix\SearchCore\Connection\Elasticsearch\Query;
 use Codappix\SearchCore\Connection\SearchRequestInterface;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
 use TYPO3\CMS\Extbase\Utility\ArrayUtility;
 
 class QueryFactory
@@ -109,20 +110,18 @@ class QueryFactory
             return;
         }
 
-        $query = ArrayUtility::setValueByPath(
+        $matchExpression = [
-            $query,
+            'type' => 'most_fields',
-            'query.bool.must.0.match._all.query',
+            'query' => $searchRequest->getSearchTerm(),
-            $searchRequest->getSearchTerm()
+            'fields' => GeneralUtility::trimExplode(',', $this->configuration->get('searching.fields')),
-        );
+        ];
 
         $minimumShouldMatch = $this->configuration->getIfExists('searching.minimumShouldMatch');
         if ($minimumShouldMatch) {
-            $query = ArrayUtility::setValueByPath(
+            $matchExpression['minimum_should_match'] = $minimumShouldMatch;
-                $query,
-                'query.bool.must.0.match._all.minimum_should_match',
-                $minimumShouldMatch
-            );
         }
+
+        $query = ArrayUtility::setValueByPath($query, 'query.bool.must.0.multi_match', $matchExpression);
     }
 
     /**
@@ -150,6 +149,7 @@ class QueryFactory
             ];
         }
 
+        if (!empty($boostQueryParts)) {
             $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [
                 'query' => [
                     'bool' => [
@@ -158,6 +158,7 @@ class QueryFactory
                 ],
             ]);
         }
+    }
 
     /**
      * @param array $query

Tests/Functional/Connection/Elasticsearch/FilterTest.php

@@ -48,7 +48,7 @@ class FilterTest extends AbstractFunctionalTestCase
 
         $searchService = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class)
             ->get(SearchService::class);
-        $searchRequest = new SearchRequest('Search Word');
+        $searchRequest = new SearchRequest();
 
         $result = $searchService->search($searchRequest);
         $this->assertSame(2, count($result), 'Did not receive both indexed elements without filter.');
@@ -73,7 +73,7 @@ class FilterTest extends AbstractFunctionalTestCase
         $searchService = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class)
             ->get(SearchService::class);
 
-        $searchRequest = new SearchRequest('Search Word');
+        $searchRequest = new SearchRequest();
         $result = $searchService->search($searchRequest);
 
         $this->assertSame(1, count($result->getFacets()), 'Did not receive the single defined facet.');

Tests/Functional/Fixtures/BasicSetup.ts

@@ -37,6 +37,7 @@ plugin {
             }
 
             searching {
+                fields = search_title
                 facets {
                     contentTypes {
                         field = CType

Tests/Unit/Connection/Elasticsearch/IndexFactoryTest.php

@@ -38,6 +38,7 @@ class IndexFactoryTest extends AbstractUnitTestCase
 
         $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock();
         $this->subject = new IndexFactory($this->configuration);
+        $this->subject->injectLogger($this->getMockedLogger());
     }
 
     /**

Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php

@@ -60,18 +60,10 @@ class TcaTableServiceTest extends AbstractUnitTestCase
             ->method('getIfExists')
             ->withConsecutive(['indexing.table.additionalWhereClause'], ['indexing.table.rootLineBlacklist'])
             ->will($this->onConsecutiveCalls(null, false));
-        $this->subject->expects($this->once())
-            ->method('getSystemWhereClause')
-            ->will($this->returnValue('1=1 AND pages.no_search = 0'));
 
-        $whereClause = $this->subject->getWhereClause();
         $this->assertSame(
             '1=1 AND pages.no_search = 0',
-            $whereClause->getStatement()
+            $this->subject->getWhereClause()
-        );
-        $this->assertSame(
-            [],
-            $whereClause->getParameters()
         );
     }
 
@@ -84,18 +76,10 @@ class TcaTableServiceTest extends AbstractUnitTestCase
             ->method('getIfExists')
             ->withConsecutive(['indexing.table.additionalWhereClause'], ['indexing.table.rootLineBlacklist'])
             ->will($this->onConsecutiveCalls('table.field = "someValue"', false));
-        $this->subject->expects($this->once())
-            ->method('getSystemWhereClause')
-            ->will($this->returnValue('1=1 AND pages.no_search = 0'));
 
-        $whereClause = $this->subject->getWhereClause();
         $this->assertSame(
             '1=1 AND pages.no_search = 0 AND table.field = "someValue"',
-            $whereClause->getStatement()
+            $this->subject->getWhereClause()
-        );
-        $this->assertSame(
-            [],
-            $whereClause->getParameters()
         );
     }
 }

Tests/Unit/Domain/Search/QueryFactoryTest.php

@@ -50,13 +50,11 @@ class QueryFactoryTest extends AbstractUnitTestCase
     /**
      * @test
      */
-    public function creatonOfQueryWorksInGeneral()
+    public function creationOfQueryWorksInGeneral()
     {
-        $searchRequest = new SearchRequest('SearchWord');
+        $this->mockConfiguration();
 
-        $this->configuration->expects($this->any())
+        $searchRequest = new SearchRequest('SearchWord');
-            ->method('get')
-            ->will($this->throwException(new InvalidArgumentException));
 
         $query = $this->subject->create($searchRequest);
         $this->assertInstanceOf(
@@ -71,9 +69,7 @@ class QueryFactoryTest extends AbstractUnitTestCase
      */
     public function filterIsAddedToQuery()
     {
-        $this->configuration->expects($this->any())
+        $this->mockConfiguration();
-            ->method('get')
-            ->will($this->throwException(new InvalidArgumentException));
 
         $searchRequest = new SearchRequest('SearchWord');
         $searchRequest->setFilter(['field' => 'content']);
@@ -83,7 +79,7 @@ class QueryFactoryTest extends AbstractUnitTestCase
             [
                 ['term' => ['field' => 'content']]
             ],
-            $query->toArray()['query']['bool']['filter'],
+            $query->toArray()['query']['function_score']['query']['bool']['filter'],
             'Filter was not added to query.'
         );
     }
@@ -93,9 +89,7 @@ class QueryFactoryTest extends AbstractUnitTestCase
      */
     public function emptyFilterIsNotAddedToQuery()
     {
-        $this->configuration->expects($this->any())
+        $this->mockConfiguration();
-            ->method('get')
-            ->will($this->throwException(new InvalidArgumentException));
 
         $searchRequest = new SearchRequest('SearchWord');
         $searchRequest->setFilter([
@@ -112,7 +106,7 @@ class QueryFactoryTest extends AbstractUnitTestCase
         $query = $this->subject->create($searchRequest);
         $this->assertSame(
             null,
-            $query->toArray()['query']['bool']['filter'],
+            $query->toArray()['query']['function_score']['query']['bool']['filter'],
             'Filter was added to query, even if no filter exists.'
         );
     }
@@ -122,9 +116,8 @@ class QueryFactoryTest extends AbstractUnitTestCase
      */
     public function facetsAreAddedToQuery()
     {
-        $this->configuration->expects($this->any())
+        $this->mockConfiguration();
-            ->method('get')
+
-            ->will($this->throwException(new InvalidArgumentException));
         $searchRequest = new SearchRequest('SearchWord');
         $searchRequest->addFacet(new FacetRequest('Identifier', 'FieldName'));
         $searchRequest->addFacet(new FacetRequest('Identifier 2', 'FieldName 2'));
@@ -153,9 +146,8 @@ class QueryFactoryTest extends AbstractUnitTestCase
      */
     public function sizeIsAddedToQuery()
     {
-        $this->configuration->expects($this->any())
+        $this->mockConfiguration();
-            ->method('get')
+
-            ->will($this->throwException(new InvalidArgumentException));
         $searchRequest = new SearchRequest('SearchWord');
         $searchRequest->setLimit(45);
         $searchRequest->setOffset(35);
@@ -178,10 +170,9 @@ class QueryFactoryTest extends AbstractUnitTestCase
      */
     public function searchTermIsAddedToQuery()
     {
+        $this->mockConfiguration();
+
         $searchRequest = new SearchRequest('SearchWord');
-        $this->configuration->expects($this->any())
-            ->method('get')
-            ->will($this->throwException(new InvalidArgumentException));
         $query = $this->subject->create($searchRequest);
 
         $this->assertSame(
@@ -189,16 +180,18 @@ class QueryFactoryTest extends AbstractUnitTestCase
                 'bool' => [
                     'must' => [
                         [
-                            'match' => [
+                            'multi_match' => [
-                                '_all' => [
+                                'type' => 'most_fields',
                                 'query' => 'SearchWord',
+                                'fields' => [
+                                    0 => 'test_field',
                                 ],
                             ],
                         ],
                     ],
                 ],
             ],
-            $query->toArray()['query'],
+            $query->toArray()['query']['function_score']['query'],
             'Search term was not added to query as expected.'
         );
     }
@@ -208,14 +201,13 @@ class QueryFactoryTest extends AbstractUnitTestCase
      */
     public function minimumShouldMatchIsAddedToQuery()
     {
-        $searchRequest = new SearchRequest('SearchWord');
         $this->configuration->expects($this->once())
             ->method('getIfExists')
             ->with('searching.minimumShouldMatch')
             ->willReturn('50%');
-        $this->configuration->expects($this->any())
+        $this->mockConfiguration();
-            ->method('get')
+
-            ->will($this->throwException(new InvalidArgumentException));
+        $searchRequest = new SearchRequest('SearchWord');
         $query = $this->subject->create($searchRequest);
 
         $this->assertArraySubset(
@@ -223,16 +215,14 @@ class QueryFactoryTest extends AbstractUnitTestCase
                 'bool' => [
                     'must' => [
                         [
-                            'match' => [
+                            'multi_match' => [
-                                '_all' => [
                                 'minimum_should_match' => '50%',
                             ],
                         ],
                     ],
                 ],
             ],
-            ],
+            $query->toArray()['query']['function_score']['query'],
-            $query->toArray()['query'],
             'minimum_should_match was not added to query as configured.'
         );
     }
@@ -244,10 +234,11 @@ class QueryFactoryTest extends AbstractUnitTestCase
     {
         $searchRequest = new SearchRequest('SearchWord');
 
-        $this->configuration->expects($this->exactly(2))
+        $this->configuration->expects($this->exactly(3))
             ->method('get')
-            ->withConsecutive(['searching.boost'], ['searching.fieldValueFactor'])
+            ->withConsecutive(['searching.fields'], ['searching.boost'], ['searching.fieldValueFactor'])
             ->will($this->onConsecutiveCalls(
+                'test_field',
                 [
                     'search_title' => 3,
                     'search_abstract' => 1.5,
@@ -292,10 +283,11 @@ class QueryFactoryTest extends AbstractUnitTestCase
             'factor' => '2',
             'missing' => '1',
         ];
-        $this->configuration->expects($this->exactly(2))
+        $this->configuration->expects($this->exactly(3))
             ->method('get')
-            ->withConsecutive(['searching.boost'], ['searching.fieldValueFactor'])
+            ->withConsecutive(['searching.fields'], ['searching.boost'], ['searching.fieldValueFactor'])
             ->will($this->onConsecutiveCalls(
+                'test_field',
                 $this->throwException(new InvalidArgumentException),
                 $fieldConfig
             ));
@@ -308,9 +300,11 @@ class QueryFactoryTest extends AbstractUnitTestCase
                         'bool' => [
                             'must' => [
                                 [
-                                    'match' => [
+                                    'multi_match' => [
-                                        '_all' => [
+                                        'type' => 'most_fields',
                                         'query' => 'SearchWord',
+                                        'fields' => [
+                                            0 => 'test_field',
                                         ],
                                     ],
                                 ],
@@ -330,17 +324,27 @@ class QueryFactoryTest extends AbstractUnitTestCase
      */
     public function emptySearchStringWillNotAddSearchToQuery()
     {
+        $this->mockConfiguration();
+
         $searchRequest = new SearchRequest();
 
-        $this->configuration->expects($this->any())
-            ->method('get')
-            ->will($this->throwException(new InvalidArgumentException));
-
         $query = $this->subject->create($searchRequest);
-        $this->assertInstanceOf(
+        $this->assertNull(
-            stdClass,
+            $query->toArray()['query']['function_score']['query'],
-            $query->toArray()['query']['match_all'],
             'Empty search request does not create expected query.'
         );
     }
+
+    protected function mockConfiguration()
+    {
+        $this->configuration->expects($this->any())
+            ->method('get')
+            ->will($this->returnCallback(function ($option) {
+                if ($option === 'searching.fields') {
+                    return 'test_field';
+                }
+
+                return $this->throwException(new InvalidArgumentException);
+            }));
+    }
 }

ext_localconf.php

@@ -11,7 +11,7 @@ call_user_func(
                 'SC_OPTIONS' => [
                     'extbase' => [
                         'commandControllers' => [
-                            Codappix\SearchCore\Command\IndexCommandController::class,
+                            $extensionKey . '::index' => Codappix\SearchCore\Command\IndexCommandController::class,
                         ],
                     ],
                     't3lib/class.t3lib_tcemain.php' => [