Merge remote-tracking branch 'origin/feature/merge-with-features' into support/76

.travis.yml

@@ -11,7 +11,6 @@ before_install:
 language: php
 
 php:
-  - 5.6
   - 7.0
   - 7.1
   - 7.2
@@ -26,7 +25,6 @@ env:
     - typo3DatabaseHost="127.0.0.1"
     - typo3DatabaseUsername="travis"
     - typo3DatabasePassword=""
-    - TYPO3_VERSION="~7.6"
 
 matrix:
   fast_finish: true

Classes/Command/IndexCommandController.php

@@ -50,7 +50,6 @@ class IndexCommandController extends CommandController
      */
     public function indexCommand($identifier)
     {
-        // TODO: Also allow to index everything?
         try {
             $this->indexerFactory->getIndexer($identifier)->indexAllDocuments();
             $this->outputLine($identifier . ' was indexed.');
@@ -58,4 +57,19 @@ class IndexCommandController extends CommandController
             $this->outputLine('No indexer found for: ' . $identifier);
         }
     }
+
+    /**
+     * Will delete the given identifier.
+     *
+     * @param string $identifier
+     */
+    public function deleteCommand($identifier)
+    {
+        try {
+            $this->indexerFactory->getIndexer($identifier)->delete();
+            $this->outputLine($identifier . ' was deleted.');
+        } catch (NoMatchingIndexerException $e) {
+            $this->outputLine('No indexer found for: ' . $identifier);
+        }
+    }
 }

Classes/Configuration/ConfigurationUtility.php

@@ -0,0 +1,69 @@
+<?php
+namespace Codappix\SearchCore\Configuration;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\SearchCore\Connection\SearchRequestInterface;
+use TYPO3\CMS\Fluid\View\StandaloneView;
+
+class ConfigurationUtility
+{
+    /**
+     * Will parse all entries, recursive as fluid template, with request variable set to $searchRequest.
+     */
+    public function replaceArrayValuesWithRequestContent(SearchRequestInterface $searchRequest, array $array) : array
+    {
+        array_walk_recursive($array, function (&$value, $key, SearchRequestInterface $searchRequest) {
+            $template = new StandaloneView();
+            $template->assign('request', $searchRequest);
+            $template->setTemplateSource($value);
+            $value = $template->render();
+
+            // As elasticsearch does need some doubles to be send as doubles.
+            if (is_numeric($value)) {
+                $value = (float) $value;
+            }
+        }, $searchRequest);
+
+        return $array;
+    }
+
+    /**
+     * Will check all entries, whether they have a condition and filter entries out, where condition is false.
+     * Also will remove condition in the end.
+     */
+    public function filterByCondition(array $entries) : array
+    {
+        $entries = array_filter($entries, function ($entry) {
+            return !is_array($entry)
+                || !array_key_exists('condition', $entry)
+                || (bool) $entry['condition'] === true
+                ;
+        });
+
+        foreach ($entries as $key => $entry) {
+            if (is_array($entry) && array_key_exists('condition', $entry)) {
+                unset($entries[$key]['condition']);
+            }
+        }
+
+        return $entries;
+    }
+}

Classes/Connection/ConnectionInterface.php

@@ -77,4 +77,13 @@ interface ConnectionInterface
      * @return SearchResultInterface
      */
     public function search(SearchRequestInterface $searchRequest);
+
+    /**
+     * Will delete the whole index / db.
+     *
+     * @param string $documentType
+     *
+     * @return void
+     */
+    public function deleteIndex($documentType);
 }

Classes/Connection/Elasticsearch.php

@@ -156,6 +156,18 @@ class Elasticsearch implements Singleton, ConnectionInterface
         );
     }
 
+    public function deleteIndex($documentType)
+    {
+        $index = $this->connection->getClient()->getIndex('typo3content');
+
+        if (! $index->exists()) {
+            $this->logger->notice('Index did not exist, therefore was not deleted.', [$documentType, 'typo3content']);
+            return;
+        }
+
+        $index->delete();
+    }
+
     /**
      * Execute given callback with Elastica Type based on provided documentType
      *

Classes/Controller/SearchController.php

@@ -56,6 +56,12 @@ class SearchController extends ActionController
                 ]
             ));
         }
+
+        if ($this->arguments->hasArgument('searchRequest')) {
+            $this->arguments->getArgument('searchRequest')->getPropertyMappingConfiguration()
+                ->allowAllProperties()
+                ;
+        }
     }
 
     /**

Classes/DataProcessing/CopyToProcessor.php

@@ -25,7 +25,7 @@ namespace Codappix\SearchCore\DataProcessing;
  */
 class CopyToProcessor implements ProcessorInterface
 {
-    public function processRecord(array $record, array $configuration)
+    public function processRecord(array $record, array $configuration) : array
     {
         $all = [];
 
@@ -42,7 +42,7 @@ class CopyToProcessor implements ProcessorInterface
      */
     protected function addArray(array &$to, array $from)
     {
-        foreach ($from as $property => $value) {
+        foreach ($from as $value) {
             if (is_array($value)) {
                 $this->addArray($to, $value);
                 continue;

Classes/DataProcessing/GeoPointProcessor.php

@@ -0,0 +1,59 @@
+<?php
+namespace Codappix\SearchCore\DataProcessing;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * Adds a new fields, ready to use as GeoPoint field for Elasticsearch.
+ */
+class GeoPointProcessor implements ProcessorInterface
+{
+    public function processRecord(array $record, array $configuration) : array
+    {
+        if (! $this->canApply($record, $configuration)) {
+            return $record;
+        }
+
+        $record[$configuration['to']] = [
+            'lat' => (float) $record[$configuration['lat']],
+            'lon' => (float) $record[$configuration['lon']],
+        ];
+
+        return $record;
+    }
+
+    protected function canApply(array $record, array $configuration) : bool
+    {
+        if (!isset($record[$configuration['lat']])
+            || !is_numeric($record[$configuration['lat']])
+            || trim($record[$configuration['lat']]) === ''
+        ) {
+            return false;
+        }
+        if (!isset($record[$configuration['lon']])
+            || !is_numeric($record[$configuration['lon']])
+            || trim($record[$configuration['lon']]) === ''
+        ) {
+            return false;
+        }
+
+        return true;
+    }
+}

Classes/DataProcessing/ProcessorInterface.php

@@ -21,19 +21,14 @@ namespace Codappix\SearchCore\DataProcessing;
  */
 
 /**
- * All DataProcessing Processor should implement this interface, otherwise they
+ * All DataProcessing Processors should implement this interface, otherwise
- * will not be executed.
+ * they will not be executed.
  */
 interface ProcessorInterface
 {
     /**
      * Processes the given record.
      * Also retrieves the configuration for this processor instance.
-     *
-     * @param array $record
-     * @param array $configuration
-     *
-     * @return array
      */
-    public function processRecord(array $record, array $configuration);
+    public function processRecord(array $record, array $configuration) : array;
 }

Classes/DataProcessing/RemoveProcessor.php

@@ -0,0 +1,44 @@
+<?php
+namespace Codappix\SearchCore\DataProcessing;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+/**
+ * Removes fields from record.
+ */
+class RemoveProcessor implements ProcessorInterface
+{
+    public function processRecord(array $record, array $configuration) : array
+    {
+        if (!isset($configuration['fields'])) {
+            return $record;
+        }
+
+        foreach (GeneralUtility::trimExplode(',', $configuration['fields'], true) as $field) {
+            if (array_key_exists($field, $record)) {
+                unset($record[$field]);
+            }
+        }
+
+        return $record;
+    }
+}

Classes/Database/Doctrine/Join.php

@@ -0,0 +1,50 @@
+<?php
+namespace Codappix\SearchCore\Database\Doctrine;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+class Join
+{
+    /**
+     * @var string
+     */
+    protected $table = '';
+
+    /**
+     * @var string
+     */
+    protected $condition = '';
+
+    public function __construct(string $table, string $condition)
+    {
+        $this->table = $table;
+        $this->condition = $condition;
+    }
+
+    public function getTable() : string
+    {
+        return $this->table;
+    }
+
+    public function getCondition() : string
+    {
+        return $this->condition;
+    }
+}

Classes/Database/Doctrine/Where.php

@@ -0,0 +1,50 @@
+<?php
+namespace Codappix\SearchCore\Database\Doctrine;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+class Where
+{
+    /**
+     * @var string
+     */
+    protected $statement = '';
+
+    /**
+     * @var array
+     */
+    protected $parameters = [];
+
+    public function __construct(string $statement, array $parameters)
+    {
+        $this->statement = $statement;
+        $this->parameters = $parameters;
+    }
+
+    public function getStatement() : string
+    {
+        return $this->statement;
+    }
+
+    public function getParameters() : array
+    {
+        return $this->parameters;
+    }
+}

Classes/Domain/Index/AbstractIndexer.php

@@ -23,7 +23,8 @@ namespace Codappix\SearchCore\Domain\Index;
 use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
 use Codappix\SearchCore\Configuration\InvalidArgumentException;
 use Codappix\SearchCore\Connection\ConnectionInterface;
-use \TYPO3\CMS\Core\Utility\GeneralUtility;
+use Codappix\SearchCore\DataProcessing\ProcessorInterface;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
 
 abstract class AbstractIndexer implements IndexerInterface
 {
@@ -47,6 +48,11 @@ abstract class AbstractIndexer implements IndexerInterface
      */
     protected $logger;
 
+    /**
+     * @var ObjectManagerInterface
+     */
+    protected $objectManager;
+
     /**
      * Inject log manager to get concrete logger from it.
      *
@@ -62,10 +68,6 @@ abstract class AbstractIndexer implements IndexerInterface
         $this->identifier = $identifier;
     }
 
-    /**
-     * @param ConnectionInterface $connection
-     * @param ConfigurationContainerInterface $configuration
-     */
     public function __construct(ConnectionInterface $connection, ConfigurationContainerInterface $configuration)
     {
         $this->connection = $connection;
@@ -99,11 +101,19 @@ abstract class AbstractIndexer implements IndexerInterface
 
             $this->connection->addDocument($this->getDocumentName(), $record);
         } catch (NoRecordFoundException $e) {
-            $this->logger->info('Could not index document.', [$e->getMessage()]);
+            $this->logger->info('Could not index document. Try to delete it therefore.', [$e->getMessage()]);
+            $this->connection->deleteDocument($this->getDocumentName(), $identifier);
         }
         $this->logger->info('Finish indexing');
     }
 
+    public function delete()
+    {
+        $this->logger->info('Start deletion of index.');
+        $this->connection->deleteIndex($this->getDocumentName());
+        $this->logger->info('Finish deletion.');
+    }
+
     /**
      * @return \Generator
      */
@@ -122,6 +132,33 @@ abstract class AbstractIndexer implements IndexerInterface
      * @param array &$record
      */
     protected function prepareRecord(array &$record)
+    {
+        try {
+            foreach ($this->configuration->get('indexing.' . $this->identifier . '.dataProcessing') as $configuration) {
+                $className = '';
+                if (is_string($configuration)) {
+                    $className = $configuration;
+                    $configuration = [];
+                } else {
+                    $className = $configuration['_typoScriptNodeValue'];
+                }
+
+                $dataProcessor = GeneralUtility::makeInstance($className);
+                if ($dataProcessor instanceof ProcessorInterface) {
+                    $record = $dataProcessor->processRecord($record, $configuration);
+                }
+            }
+        } catch (InvalidArgumentException $e) {
+            // Nothing to do.
+        }
+
+        $this->handleAbstract($record);
+    }
+
+    /**
+     * @param array &$record
+     */
+    protected function handleAbstract(array &$record)
     {
         $record['search_abstract'] = '';
 

Classes/Domain/Index/IndexerFactory.php

@@ -54,12 +54,9 @@ class IndexerFactory implements Singleton
     }
 
     /**
-     * @param string $identifier
-     *
-     * @return IndexerInterface
      * @throws NoMatchingIndexer
      */
-    public function getIndexer($identifier)
+    public function getIndexer(string $identifier) : IndexerInterface
     {
         try {
             return $this->buildIndexer($this->configuration->get('indexing.' . $identifier . '.indexer'), $identifier);
@@ -73,13 +70,9 @@ class IndexerFactory implements Singleton
     }
 
     /**
-     * @param string $indexerClass
-     * @param string $identifier
-     *
-     * @return IndexerInterface
      * @throws NoMatchingIndexer
      */
-    protected function buildIndexer($indexerClass, $identifier)
+    protected function buildIndexer(string $indexerClass, string $identifier) : IndexerInterface
     {
         $indexer = null;
         if (is_subclass_of($indexerClass, TcaIndexer\PagesIndexer::class)

Classes/Domain/Index/IndexerInterface.php

@@ -33,9 +33,9 @@ interface IndexerInterface
     public function indexAllDocuments();
 
     /**
-     * Fetches a single document from the indexerService and pushes it to the connection.
+     * Fetches a single document and pushes it to the connection.
      *
-     * @param string $identifier identifier, the indexer needs to identify a single document
+     * @param string $identifier
      *
      * @return void
      */
@@ -49,4 +49,11 @@ interface IndexerInterface
      * @return void
      */
     public function setIdentifier($identifier);
+
+    /**
+     * Delete the whole index.
+     *
+     * @return void
+     */
+    public function delete();
 }

Classes/Domain/Index/TcaIndexer/PagesIndexer.php

@@ -58,9 +58,6 @@ class PagesIndexer extends TcaIndexer
         $this->configuration = $configuration;
     }
 
-    /**
-     * @param array &$record
-     */
     protected function prepareRecord(array &$record)
     {
         $possibleTitleFields = ['nav_title', 'tx_tqseo_pagetitle_rel', 'title'];
@@ -80,11 +77,7 @@ class PagesIndexer extends TcaIndexer
         parent::prepareRecord($record);
     }
 
-    /**
+    protected function fetchContentForPage(int $uid) : array
-     * @param int $uid
-     * @return []
-     */
-    protected function fetchContentForPage($uid)
     {
         $contentElements = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows(
             $this->contentTableService->getFields(),
@@ -118,31 +111,17 @@ class PagesIndexer extends TcaIndexer
         ];
     }
 
-    /**
+    protected function getContentElementImages(int $uidOfContentElement) : array
-     * @param int $uidOfContentElement
-     * @return array
-     */
-    protected function getContentElementImages($uidOfContentElement)
     {
         return $this->fetchSysFileReferenceUids($uidOfContentElement, 'tt_content', 'image');
     }
 
-    /**
+    protected function fetchMediaForPage(int $uid) : array
-     * @param int $uid
-     * @return []
-     */
-    protected function fetchMediaForPage($uid)
     {
         return $this->fetchSysFileReferenceUids($uid, 'pages', 'media');
     }
 
-    /**
+    protected function fetchSysFileReferenceUids(int $uid, string $tablename, string $fieldname) : array
-     * @param int $uid
-     * @param string $tablename
-     * @param string $fieldname
-     * @return []
-     */
-    protected function fetchSysFileReferenceUids($uid, $tablename, $fieldname)
     {
         $imageRelationUids = [];
         $imageRelations = $this->fileRepository->findByRelation($tablename, $fieldname, $uid);

Classes/Domain/Index/TcaIndexer/RelationResolver.php

@@ -20,8 +20,10 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer;
  * 02110-1301, USA.
  */
 
+use Codappix\SearchCore\Utility\FrontendUtility;
 use TYPO3\CMS\Backend\Utility\BackendUtility;
 use TYPO3\CMS\Core\SingletonInterface as Singleton;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
 
 /**
  * Resolves relations from TCA using TCA.
@@ -38,7 +40,9 @@ class RelationResolver implements Singleton
             if ($column === 'pid') {
                 continue;
             }
-            $record[$column] = BackendUtility::getProcessedValueExtra(
+
+            $record[$column] = GeneralUtility::makeInstance($this->getUtilityForMode())
+                ::getProcessedValueExtra(
                     $service->getTableName(),
                     $column,
                     $record[$column],
@@ -75,7 +79,7 @@ class RelationResolver implements Singleton
         return [];
     }
 
-    protected function isRelation(array &$config)
+    protected function isRelation(array &$config) : bool
     {
         return isset($config['foreign_table'])
             || (isset($config['renderType']) && $config['renderType'] !== 'selectSingle')
@@ -83,13 +87,22 @@ class RelationResolver implements Singleton
             ;
     }
 
-    protected function resolveForeignDbValue($value)
+    protected function resolveForeignDbValue(string $value) : array
     {
         return array_map('trim', explode(';', $value));
     }
 
-    protected function resolveInlineValue($value)
+    protected function resolveInlineValue(string $value) : array
     {
         return array_map('trim', explode(',', $value));
     }
+
+    protected function getUtilityForMode() : string
+    {
+        if (TYPO3_MODE === 'BE') {
+            return BackendUtility::class;
+        }
+
+        return FrontendUtility::class;
+    }
 }

Classes/Domain/Index/TcaIndexer/TcaTableService.php

@@ -22,7 +22,6 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer;
 
 use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
 use Codappix\SearchCore\Configuration\InvalidArgumentException as InvalidConfigurationArgumentException;
-use Codappix\SearchCore\DataProcessing\ProcessorInterface;
 use Codappix\SearchCore\Domain\Index\IndexingException;
 use TYPO3\CMS\Backend\Utility\BackendUtility;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
@@ -109,7 +108,7 @@ class TcaTableService
     /**
      * @return string
      */
-    public function getTableName()
+    public function getTableName() : string
     {
         return $this->tableName;
     }
@@ -117,7 +116,7 @@ class TcaTableService
     /**
      * @return string
      */
-    public function getTableClause()
+    public function getTableClause() : string
     {
         if ($this->tableName === 'pages') {
             return $this->tableName;
@@ -128,9 +127,6 @@ class TcaTableService
 
     /**
      * Filter the given records by root line blacklist settings.
-     *
-     * @param array &$records
-     * @return void
      */
     public function filterRecordsByRootLineBlacklist(array &$records)
     {
@@ -144,23 +140,11 @@ class TcaTableService
 
     /**
      * Adjust record accordingly to configuration.
-     * @param array &$record
      */
     public function prepareRecord(array &$record)
     {
         $this->relationResolver->resolveRelationsForRecord($this, $record);
 
-        try {
-            foreach ($this->configuration->get('indexing.' . $this->tableName . '.dataProcessing') as $configuration) {
-                $dataProcessor = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance($configuration['_typoScriptNodeValue']);
-                if ($dataProcessor instanceof ProcessorInterface) {
-                    $record = $dataProcessor->processRecord($record, $configuration);
-                }
-            }
-        } catch (InvalidConfigurationArgumentException $e) {
-            // Nothing to do.
-        }
-
         if (isset($record['uid']) && !isset($record['search_identifier'])) {
             $record['search_identifier'] = $record['uid'];
         }
@@ -169,10 +153,7 @@ class TcaTableService
         }
     }
 
-    /**
+    public function getWhereClause() : string
-     * @return string
-     */
-    public function getWhereClause()
     {
         $whereClause = '1=1'
             . BackendUtility::BEenableFields($this->tableName)
@@ -204,17 +185,17 @@ class TcaTableService
         return $whereClause;
     }
 
-    /**
+    public function getFields() : string
-     * @return string
-     */
-    public function getFields()
     {
         $fields = array_merge(
             ['uid','pid'],
             array_filter(
                 array_keys($this->tca['columns']),
                 function ($columnName) {
-                    return !$this->isSystemField($columnName);
+                    return !$this->isSystemField($columnName)
+                        && !$this->isUserField($columnName)
+                        && !$this->isPassthroughField($columnName)
+                        ;
                 }
             )
         );
@@ -228,10 +209,27 @@ class TcaTableService
     }
 
     /**
-     * @param string
+     * Generate SQL for TYPO3 as a system, to make sure only available records
-     * @return bool
+     * are fetched.
      */
-    protected function isSystemField($columnName)
+    public function getSystemWhereClause() : string
+    {
+        $whereClause = '1=1'
+            . BackendUtility::BEenableFields($this->tableName)
+            . BackendUtility::deleteClause($this->tableName)
+            . ' AND pages.no_search = 0'
+            ;
+
+        if ($this->tableName !== 'pages') {
+            $whereClause .= BackendUtility::BEenableFields('pages')
+                . BackendUtility::deleteClause('pages')
+            ;
+        }
+
+        return $whereClause;
+    }
+
+    protected function isSystemField(string $columnName) : bool
     {
         $systemFields = [
             // Versioning fields,
@@ -250,12 +248,22 @@ class TcaTableService
         return in_array($columnName, $systemFields);
     }
 
+    protected function isUserField(string $columnName) : bool
+    {
+        $config = $this->getColumnConfig($columnName);
+        return isset($config['type']) && $config['type'] === 'user';
+    }
+
+    protected function isPassthroughField(string $columnName) : bool
+    {
+        $config = $this->getColumnConfig($columnName);
+        return isset($config['type']) && $config['type'] === 'passthrough';
+    }
+
     /**
-     * @param string $columnName
-     * @return array
      * @throws InvalidArgumentException
      */
-    public function getColumnConfig($columnName)
+    public function getColumnConfig(string $columnName) : array
     {
         if (!isset($this->tca['columns'][$columnName])) {
             throw new InvalidArgumentException(
@@ -274,11 +282,8 @@ class TcaTableService
      * Also further TYPO3 mechanics are taken into account. Does a valid root
      * line exist, is page inside a recycler, is inherited start- endtime
      * excluded, etc.
-     *
-     * @param array &$record
-     * @return bool
      */
-    protected function isRecordBlacklistedByRootline(array &$record)
+    protected function isRecordBlacklistedByRootline(array &$record) : bool
     {
         $pageUid = $record['pid'];
         if ($this->tableName === 'pages') {
@@ -332,20 +337,16 @@ class TcaTableService
 
     /**
      * Checks whether any page uids are black listed.
-     *
-     * @return bool
      */
-    protected function isBlackListedRootLineConfigured()
+    protected function isBlackListedRootLineConfigured() : bool
     {
         return (bool) $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist');
     }
 
     /**
      * Get the list of black listed root line page uids.
-     *
-     * @return array<Int>
      */
-    protected function getBlackListedRootLine()
+    protected function getBlackListedRootLine() : array
     {
         return GeneralUtility::intExplode(',', $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist'));
     }

Classes/Domain/Model/SearchRequest.php

@@ -92,7 +92,8 @@ class SearchRequest implements SearchRequestInterface
      */
     public function setFilter(array $filter)
     {
-        $this->filter = array_filter(array_map('strval', $filter));
+        $filter = \TYPO3\CMS\Core\Utility\ArrayUtility::removeArrayEntryByValue($filter, '');
+        $this->filter = \TYPO3\CMS\Extbase\Utility\ArrayUtility::removeEmptyElementsRecursively($filter);
     }
 
     /**

Classes/Domain/Search/QueryFactory.php

@@ -21,6 +21,7 @@ namespace Codappix\SearchCore\Domain\Search;
  */
 
 use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
+use Codappix\SearchCore\Configuration\ConfigurationUtility;
 use Codappix\SearchCore\Configuration\InvalidArgumentException;
 use Codappix\SearchCore\Connection\Elasticsearch\Query;
 use Codappix\SearchCore\Connection\SearchRequestInterface;
@@ -40,37 +41,31 @@ class QueryFactory
     protected $configuration;
 
     /**
-     * @param \TYPO3\CMS\Core\Log\LogManager $logManager
+     * @var ConfigurationUtility
-     * @param ConfigurationContainerInterface $configuration
      */
+    protected $configurationUtility;
+
     public function __construct(
         \TYPO3\CMS\Core\Log\LogManager $logManager,
-        ConfigurationContainerInterface $configuration
+        ConfigurationContainerInterface $configuration,
+        ConfigurationUtility $configurationUtility
     ) {
         $this->logger = $logManager->getLogger(__CLASS__);
         $this->configuration = $configuration;
+        $this->configurationUtility = $configurationUtility;
     }
 
     /**
      * TODO: This is not in scope Elasticsearch, therefore it should not return
      * \Elastica\Query, but decide to use a more specific QueryFactory like
      * ElasticaQueryFactory, once the second query is added?
-     *
-     * @param SearchRequestInterface $searchRequest
-     *
-     * @return \Elastica\Query
      */
-    public function create(SearchRequestInterface $searchRequest)
+    public function create(SearchRequestInterface $searchRequest) : \Elastica\Query
     {
         return $this->createElasticaQuery($searchRequest);
     }
 
-    /**
+    protected function createElasticaQuery(SearchRequestInterface $searchRequest) : \Elastica\Query
-     * @param SearchRequestInterface $searchRequest
-     *
-     * @return \Elastica\Query
-     */
-    protected function createElasticaQuery(SearchRequestInterface $searchRequest)
     {
         $query = [];
         $this->addSize($searchRequest, $query);
@@ -78,6 +73,8 @@ class QueryFactory
         $this->addBoosts($searchRequest, $query);
         $this->addFilter($searchRequest, $query);
         $this->addFacets($searchRequest, $query);
+        $this->addFields($searchRequest, $query);
+        $this->addSort($searchRequest, $query);
 
         // Use last, as it might change structure of query.
         // Better approach would be something like DQL to generate query and build result in the end.
@@ -87,10 +84,6 @@ class QueryFactory
         return new \Elastica\Query($query);
     }
 
-    /**
-     * @param SearchRequestInterface $searchRequest
-     * @param array $query
-     */
     protected function addSize(SearchRequestInterface $searchRequest, array &$query)
     {
         $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [
@@ -99,10 +92,6 @@ class QueryFactory
         ]);
     }
 
-    /**
-     * @param SearchRequestInterface $searchRequest
-     * @param array $query
-     */
     protected function addSearch(SearchRequestInterface $searchRequest, array &$query)
     {
         if (trim($searchRequest->getSearchTerm()) === '') {
@@ -112,7 +101,7 @@ class QueryFactory
         $matchExpression = [
             'type' => 'most_fields',
             'query' => $searchRequest->getSearchTerm(),
-            'fields' => GeneralUtility::trimExplode(',', $this->configuration->get('searching.fields')),
+            'fields' => GeneralUtility::trimExplode(',', $this->configuration->get('searching.fields.query')),
         ];
 
         $minimumShouldMatch = $this->configuration->getIfExists('searching.minimumShouldMatch');
@@ -123,10 +112,6 @@ class QueryFactory
         $query = ArrayUtility::setValueByPath($query, 'query.bool.must.0.multi_match', $matchExpression);
     }
 
-    /**
-     * @param SearchRequestInterface $searchRequest
-     * @param array $query
-     */
     protected function addBoosts(SearchRequestInterface $searchRequest, array &$query)
     {
         try {
@@ -159,9 +144,6 @@ class QueryFactory
         }
     }
 
-    /**
-     * @param array $query
-     */
     protected function addFactorBoost(array &$query)
     {
         try {
@@ -176,38 +158,83 @@ class QueryFactory
         }
     }
 
-    /**
+    protected function addFields(SearchRequestInterface $searchRequest, array &$query)
-     * @param SearchRequestInterface $searchRequest
+    {
-     * @param array $query
+        try {
-     */
+            $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [
+                'stored_fields' => GeneralUtility::trimExplode(',', $this->configuration->get('searching.fields.stored_fields'), true),
+            ]);
+        } catch (InvalidArgumentException $e) {
+            // Nothing configured
+        }
+
+        try {
+            $scriptFields = $this->configuration->get('searching.fields.script_fields');
+            $scriptFields = $this->configurationUtility->replaceArrayValuesWithRequestContent($searchRequest, $scriptFields);
+            $scriptFields = $this->configurationUtility->filterByCondition($scriptFields);
+            if ($scriptFields !== []) {
+                $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['script_fields' => $scriptFields]);
+            }
+        } catch (InvalidArgumentException $e) {
+            // Nothing configured
+        }
+    }
+
+    protected function addSort(SearchRequestInterface $searchRequest, array &$query)
+    {
+        $sorting = $this->configuration->getIfExists('searching.sort') ?: [];
+        $sorting = $this->configurationUtility->replaceArrayValuesWithRequestContent($searchRequest, $sorting);
+        $sorting = $this->configurationUtility->filterByCondition($sorting);
+        if ($sorting !== []) {
+            $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['sort' => $sorting]);
+        }
+    }
+
     protected function addFilter(SearchRequestInterface $searchRequest, array &$query)
     {
         if (! $searchRequest->hasFilter()) {
             return;
         }
 
-        $terms = [];
+        $filter = [];
         foreach ($searchRequest->getFilter() as $name => $value) {
-            $terms[] = [
+            $filter[] = $this->buildFilter(
+                $name,
+                $value,
+                $this->configuration->getIfExists('searching.mapping.filter.' . $name) ?: []
+            );
+        }
+
+        $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [
+            'query' => [
+                'bool' => [
+                    'filter' => $filter,
+                ],
+            ],
+        ]);
+    }
+
+    protected function buildFilter(string $name, $value, array $config) : array
+    {
+        if ($config === []) {
+            return [
                 'term' => [
                     $name => $value,
                 ],
             ];
         }
 
-        $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [
+        $filter = [];
-            'query' => [
+
-                'bool' => [
+        if (isset($config['fields'])) {
-                    'filter' => $terms,
+            foreach ($config['fields'] as $elasticField => $inputField) {
-                ],
+                $filter[$elasticField] = $value[$inputField];
-            ],
+            }
-        ]);
+        }
+
+        return [$config['field'] => $filter];
     }
 
-    /**
-     * @param SearchRequestInterface $searchRequest
-     * @param array $query
-     */
     protected function addFacets(SearchRequestInterface $searchRequest, array &$query)
     {
         foreach ($searchRequest->getFacets() as $facet) {

Classes/Domain/Search/SearchService.php

@@ -26,6 +26,7 @@ use Codappix\SearchCore\Connection\ConnectionInterface;
 use Codappix\SearchCore\Connection\SearchRequestInterface;
 use Codappix\SearchCore\Connection\SearchResultInterface;
 use Codappix\SearchCore\Domain\Model\FacetRequest;
+use TYPO3\CMS\Core\Utility\ArrayUtility;
 use TYPO3\CMS\Extbase\Object\ObjectManagerInterface;
 
 /**
@@ -123,10 +124,16 @@ class SearchService
     protected function addConfiguredFilters(SearchRequestInterface $searchRequest)
     {
         try {
-            $searchRequest->setFilter(array_merge(
+            $filter = $searchRequest->getFilter();
-                $searchRequest->getFilter(),
+
-                $this->configuration->get('searching.filter')
+            ArrayUtility::mergeRecursiveWithOverrule(
-            ));
+                $filter,
+                $this->configuration->get('searching.filter'),
+                true,
+                false
+            );
+
+            $searchRequest->setFilter($filter);
         } catch (InvalidArgumentException $e) {
             // Nothing todo, no filter configured.
         }

Classes/Utility/FrontendUtility.php

@@ -0,0 +1,39 @@
+<?php
+namespace Codappix\SearchCore\Utility;
+
+/*
+ * Copyright (C) 2018 Justus Moroni <developer@leonmrni.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use TYPO3\CMS\Backend\Utility\BackendUtility;
+use TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController;
+
+/**
+ * Overwrite BackendUtility to use in frontend.
+ * LanguageService was only usable in backend.
+ */
+class FrontendUtility extends BackendUtility
+{
+    /**
+     * @return TypoScriptFrontendController
+     */
+    protected static function getLanguageService()
+    {
+        return $GLOBALS['TSFE'];
+    }
+}

Configuration/TypoScript/setup.txt

@@ -21,6 +21,12 @@ plugin {
                     abstractFields = {$plugin.tx_searchcore.settings.indexing.pages.abstractFields}
                 }
             }
+
+            searching {
+                fields {
+                    query = _all
+                }
+            }
         }
     }
 }

Documentation/source/concepts.rst

@@ -28,3 +28,13 @@ The indexing is done by one of the available indexer. For each identifier it's p
 the indexer to use. Also it's possible to write custom indexer to use.
 
 Currently only the :ref:`TcaIndexer` is provided.
+
+.. _concepts_indexing_dataprocessing:
+
+DataProcessing
+^^^^^^^^^^^^^^
+
+Before data is transfered to search service, it can be processed by "DataProcessors" like already
+known by :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing` of :ref:`t3tsref:cobj-fluidtemplate`.
+
+Configuration is done through TypoScript, see :ref:`dataProcessing`.

Documentation/source/conf.py

@@ -304,6 +304,7 @@ texinfo_documents = [
 # Example configuration for intersphinx: refer to the Python standard library.
 intersphinx_mapping = {
     't3tcaref': ('https://docs.typo3.org/typo3cms/TCAReference/', None),
+    't3tsref': ('https://docs.typo3.org/typo3cms/TyposcriptReference/', None),
 }
 extlinks = {
     'project': ('https://github.com/Codappix/search_core/projects/%s', 'Github project: '),

Documentation/source/configuration.rst

@@ -36,330 +36,14 @@ Here is the example default configuration that's provided through static include
 Options
 -------
 
-The following section contains the different options, e.g. for :ref:`connections` and
+The following sections contains the different options grouped by their applied area, e.g. for
-:ref:`indexer`: ``plugin.tx_searchcore.settings.connection`` or
+:ref:`connections` and :ref:`indexer`: ``plugin.tx_searchcore.settings.connection`` or
-``plugin.tx_searchcore.settings.indexing``.
+``plugin.tx_searchcore.settings.indexing``:
 
-.. _configuration_options_connection:
+.. toctree::
+   :maxdepth: 1
+   :glob:
 
-connections
+   configuration/connections
-^^^^^^^^^^^
+   configuration/indexing
-
+   configuration/searching
-Holds settings regarding the different possible connections for search services like Elasticsearch
-or Solr.
-
-Configured as::
-
-    plugin {
-        tx_searchcore {
-            settings {
-                connections {
-                    connectionName {
-                        // the settings
-                    }
-                }
-            }
-        }
-    }
-
-Where ``connectionName`` is one of the available :ref:`connections`.
-
-The following settings are available. For each setting its documented which connection consumes it.
-
-.. _host:
-
-``host``
-""""""""
-
-    Used by: :ref:`Elasticsearch`.
-
-    The host, e.g. ``localhost`` or an IP where the search service is reachable from TYPO3
-    installation.
-
-    Example::
-
-        plugin.tx_searchcore.settings.connections.elasticsearch.host = localhost
-
-.. _port:
-
-``port``
-""""""""
-
-    Used by: :ref:`Elasticsearch`.
-
-    The port where search service is reachable. E.g. default ``9200`` for Elasticsearch.
-
-    Example::
-
-        plugin.tx_searchcore.settings.connections.elasticsearch.port = 9200
-
-
-.. _configuration_options_index:
-
-Indexing
-^^^^^^^^
-
-Holds settings regarding the indexing, e.g. of TYPO3 records, to search services.
-
-Configured as::
-
-    plugin {
-        tx_searchcore {
-            settings {
-                indexing {
-                    identifier {
-                        indexer = FullyQualifiedClassname
-                        // the settings
-                    }
-                }
-            }
-        }
-    }
-
-Where ``identifier`` is up to you, but should match table names to make :ref:`TcaIndexer` work.
-
-The following settings are available. For each setting its documented which indexer consumes it.
-
-.. _rootLineBlacklist:
-
-``rootLineBlacklist``
-"""""""""""""""""""""
-
-    Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`.
-
-    Defines a blacklist of page uids. Records below any of these pages, or subpages, are not
-    indexed. This allows you to define areas that should not be indexed.
-    The page attribute *No Search* is also taken into account to prevent indexing records from only one
-    page without recursion.
-
-    Contains a comma separated list of page uids. Spaces are trimmed.
-
-    Example::
-
-        plugin.tx_searchcore.settings.indexing.<identifier>.rootLineBlacklist = 3, 10, 100
-
-Also it's possible to define some behaviour for the different document types. In context of TYPO3
-tables are used as document types 1:1. It's possible to configure different tables. The following
-options are available:
-
-.. _additionalWhereClause:
-
-``additionalWhereClause``
-"""""""""""""""""""""""""
-
-    Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`.
-
-    Add additional SQL to where clauses to determine indexable records from the table. This way you
-    can exclude specific records like ``tt_content`` records with specific ``CType`` values or
-    something else. E.g. you can add a new field to the table to exclude records from indexing.
-
-    Example::
-
-        plugin.tx_searchcore.settings.indexing.<identifier>.additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu')
-
-    .. attention::
-
-        Make sure to prefix all fields with the corresponding table name. The selection from
-        database will contain joins and can lead to SQL errors if a field exists in multiple tables.
-
-.. _abstractFields:
-
-``abstractFields``
-"""""""""""""""""""""""""
-
-    Used by: :ref:`PagesIndexer`.
-
-    Define which field should be used to provide the auto generated field "search_abstract".
-    The fields have to exist in the record to be indexed. Therefore fields like ``content`` are also
-    possible.
-
-    Example::
-
-        # As last fallback we use the content of the page
-        plugin.tx_searchcore.settings.indexing.<identifier>.abstractFields := addToList(content)
-
-    Default::
-
-        abstract, description, bodytext
-
-.. _mapping:
-
-``mapping``
-"""""""""""
-
-    Used by: Elasticsearch connection while indexing.
-
-    Define mapping for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/mapping.html
-    You are able to define the mapping for each property / columns.
-
-    Example::
-
-        plugin.tx_searchcore.settings.indexing.tt_content.mapping {
-            CType {
-                type = keyword
-            }
-        }
-
-    The above example will define the ``CType`` field of ``tt_content`` as ``type: keyword``. This
-    makes building a facet possible.
-
-
-.. _index:
-
-``index``
-"""""""""
-
-    Used by: Elasticsearch connection while indexing.
-
-    Define index for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/indices-create-index.html
-
-    Example::
-
-        plugin.tx_searchcore.settings.indexing.tt_content.index {
-            analysis {
-                analyzer {
-                    ngram4 {
-                        type = custom
-                        tokenizer = ngram4
-                        char_filter = html_strip
-                        filter = lowercase, asciifolding
-                    }
-                }
-
-                tokenizer {
-                    ngram4 {
-                        type = ngram
-                        min_gram = 4
-                        max_gram = 4
-                    }
-                }
-            }
-        }
-
-    ``char_filter`` and ``filter`` are a comma separated list of options.
-
-.. _configuration_options_search:
-
-Searching
-^^^^^^^^^
-
-.. _size:
-
-``size``
-""""""""
-
-    Used by: Elasticsearch connection while building search query.
-
-    Defined how many search results should be fetched to be available in search result.
-
-    Example::
-
-        plugin.tx_searchcore.settings.searching.size = 50
-
-    Default if not configured is 10.
-
-.. _facets:
-
-``facets``
-"""""""""""
-
-    Used by: Elasticsearch connection while building search query.
-
-    Define aggregations for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-aggregations-bucket-terms-aggregation.html
-    Currently only the term facet is provided.
-
-    Example::
-
-        plugin.tx_searchcore.settings.searching.facets {
-            contentTypes {
-                field = CType
-            }
-        }
-
-    The above example will provide a facet with options for all found ``CType`` results together
-    with a count.
-
-.. _filter:
-
-``filter``
-"""""""""""
-
-    Used by: While building search request.
-
-    Define filter that should be set for all requests.
-
-    Example::
-
-        plugin.tx_searchcore.settings.searching.filter {
-            property = value
-        }
-
-    For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``keyword``.
-
-.. _minimumShouldMatch:
-
-``minimumShouldMatch``
-""""""""""""""""""""""
-
-    Used by: Elasticsearch connection while building search query.
-
-    Define the minimum match for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-minimum-should-match.html
-
-    Example::
-
-        plugin.tx_searchcore.settings.searching.minimumShouldMatch = 50%
-
-.. _boost:
-
-``boost``
-"""""""""
-
-    Used by: Elasticsearch connection while building search query.
-
-    Define fields that should boost the score for results.
-
-    Example::
-
-        plugin.tx_searchcore.settings.searching.boost {
-            search_title = 3
-            search_abstract = 1.5
-        }
-
-    For further information take a look at
-    https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_boosting_query_clauses.html
-
-.. _fieldValueFactor:
-
-``fieldValueFactor``
-""""""""""""""""""""
-
-    Used by: Elasticsearch connection while building search query.
-
-    Define a field to use as a factor for scoring. The configuration is passed through to elastic
-    search ``field_value_factor``, see: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor
-
-    Example::
-
-        plugin.tx_searchcore.settings.searching.field_value_factor {
-            field = rootlineLevel
-            modifier = reciprocal
-            factor = 2
-            missing = 1
-        }
-
-.. _mode:
-
-``mode``
-""""""""
-
-    Used by: Controller while preparing action.
-
-    Define to switch from search to filter mode.
-
-    Example::
-
-        plugin.tx_searchcore.settings.searching {
-            mode = filter
-        }
-
-    Only ``filter`` is allowed as value. Will submit an empty query to switch to filter mode.

Documentation/source/configuration/connections.rst

@@ -0,0 +1,55 @@
+.. _configuration_options_connection:
+
+Connections
+===========
+
+Holds settings regarding the different possible connections for search services like Elasticsearch
+or Solr.
+
+Configured as::
+
+    plugin {
+        tx_searchcore {
+            settings {
+                connections {
+                    connectionName {
+                        // the settings
+                    }
+                }
+            }
+        }
+    }
+
+Where ``connectionName`` is one of the available :ref:`connections`.
+
+The following settings are available. For each setting its documented which connection consumes it.
+
+.. _host:
+
+``host``
+--------
+
+Used by: :ref:`Elasticsearch`.
+
+The host, e.g. ``localhost`` or an IP where the search service is reachable from TYPO3
+installation.
+
+Example::
+
+    plugin.tx_searchcore.settings.connections.elasticsearch.host = localhost
+
+.. _port:
+
+``port``
+--------
+
+Used by: :ref:`Elasticsearch`.
+
+The port where search service is reachable. E.g. default ``9200`` for Elasticsearch.
+
+Example::
+
+    plugin.tx_searchcore.settings.connections.elasticsearch.port = 9200
+
+
+

Documentation/source/configuration/dataProcessing/CopyToProcessor.rst

@@ -0,0 +1,23 @@
+``Codappix\SearchCore\DataProcessing\CopyToProcessor``
+======================================================
+
+Will copy contents of fields to other fields.
+
+Possible Options:
+
+``to``
+    Defines the field to copy the values into. All values not false will be copied at the moment.
+
+Example::
+
+    plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing {
+        1 = Codappix\SearchCore\DataProcessing\CopyToProcessor
+        1 {
+            to = all
+        }
+        2 = Codappix\SearchCore\DataProcessing\CopyToProcessor
+        2 {
+            to = spellcheck
+        }
+    }
+

Documentation/source/configuration/dataProcessing/GeoPointProcessor.rst

@@ -0,0 +1,27 @@
+``Codappix\SearchCore\DataProcessing\GeoPointProcessor``
+========================================================
+
+Will create a new field, ready to use as GeoPoint field for Elasticsearch.
+
+Possible Options:
+
+``to``
+    Defines the field to create as GeoPoint.
+``lat``
+    Defines the field containing the latitude.
+``lon``
+    Defines the field containing the longitude.
+
+Example::
+
+    plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing {
+        1 = Codappix\SearchCore\DataProcessing\GeoPointProcessor
+        1 {
+            to = location
+            lat = lat
+            lon = lng
+        }
+    }
+
+The above example will create a new field ``location`` as GeoPoint with latitude fetched from field
+``lat`` and longitude fetched from field ``lng``.

Documentation/source/configuration/dataProcessing/RemoveProcessor.rst

@@ -0,0 +1,23 @@
+``Codappix\SearchCore\DataProcessing\RemoveProcessor``
+======================================================
+
+Will remove fields from record, e.g. if you do not want to sent them to elasticsearch at all.
+
+Possible Options:
+
+``fields``
+    Comma separated list of fields to remove from record.
+
+Example::
+
+    plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing {
+        1 = Codappix\SearchCore\DataProcessing\RemoveProcessor
+        1 {
+            fields = description
+        }
+        2 = Codappix\SearchCore\DataProcessing\RemoveProcessor
+        2 {
+            fields = description, another_field
+        }
+    }
+

Documentation/source/configuration/indexing.rst

@@ -0,0 +1,207 @@
+.. _configuration_options_index:
+
+Indexing
+========
+
+Holds settings regarding the indexing, e.g. of TYPO3 records, to search services.
+
+Configured as::
+
+    plugin {
+        tx_searchcore {
+            settings {
+                indexing {
+                    identifier {
+                        indexer = FullyQualifiedClassname
+                        // the settings
+                    }
+                }
+            }
+        }
+    }
+
+Where ``identifier`` is up to you, but should match table names to make :ref:`TcaIndexer` work.
+
+The following settings are available. For each setting its documented which indexer consumes it.
+
+.. _rootLineBlacklist:
+
+rootLineBlacklist
+-----------------
+
+Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`.
+
+Defines a blacklist of page uids. Records below any of these pages, or subpages, are not
+indexed. This allows you to define areas that should not be indexed.
+The page attribute *No Search* is also taken into account to prevent indexing records from only one
+page without recursion.
+
+Contains a comma separated list of page uids. Spaces are trimmed.
+
+Example::
+
+    plugin.tx_searchcore.settings.indexing.<identifier>.rootLineBlacklist = 3, 10, 100
+
+Also it's possible to define some behaviour for the different document types. In context of TYPO3
+tables are used as document types 1:1. It's possible to configure different tables. The following
+options are available:
+
+.. _additionalWhereClause:
+
+additionalWhereClause
+---------------------
+
+Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`.
+
+Add additional SQL to where clauses to determine indexable records from the table. This way you
+can exclude specific records like ``tt_content`` records with specific ``CType`` values or
+something else. E.g. you can add a new field to the table to exclude records from indexing.
+
+Example::
+
+    plugin.tx_searchcore.settings.indexing.<identifier>.additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu')
+
+.. attention::
+
+    Make sure to prefix all fields with the corresponding table name. The selection from
+    database will contain joins and can lead to SQL errors if a field exists in multiple tables.
+
+.. _abstractFields:
+
+abstractFields
+--------------
+
+Used by: :ref:`PagesIndexer`.
+
+Define which field should be used to provide the auto generated field "search_abstract".
+The fields have to exist in the record to be indexed. Therefore fields like ``content`` are also
+possible.
+
+Example::
+
+    # As last fallback we use the content of the page
+    plugin.tx_searchcore.settings.indexing.<identifier>.abstractFields := addToList(content)
+
+Default::
+
+    abstract, description, bodytext
+
+.. _mapping:
+
+mapping
+-------
+
+Used by: Elasticsearch connection while indexing.
+
+Define mapping for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/mapping.html
+You are able to define the mapping for each property / columns.
+
+Example::
+
+    plugin.tx_searchcore.settings.indexing.tt_content.mapping {
+        CType {
+            type = keyword
+        }
+    }
+
+The above example will define the ``CType`` field of ``tt_content`` as ``type: keyword``. This
+makes building a facet possible.
+
+.. _index:
+
+index
+-----
+
+Used by: Elasticsearch connection while indexing.
+
+Define index for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/indices-create-index.html
+
+Example::
+
+    plugin.tx_searchcore.settings.indexing.tt_content.index {
+        analysis {
+            analyzer {
+                ngram4 {
+                    type = custom
+                    tokenizer = ngram4
+                    char_filter = html_strip
+                    filter = lowercase, asciifolding
+                }
+            }
+
+            tokenizer {
+                ngram4 {
+                    type = ngram
+                    min_gram = 4
+                    max_gram = 4
+                }
+            }
+        }
+    }
+
+``char_filter`` and ``filter`` are a comma separated list of options.
+
+.. _dataProcessing:
+
+dataProcessing
+--------------
+
+Used by: All connections while indexing.
+
+Configure modifications on each document before sending it to the configured connection. Same as
+provided by TYPO3 for :ref:`t3tsref:cobj-fluidtemplate` through
+:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`.
+
+All processors are applied in configured order. Allowing to work with already processed data.
+
+Example::
+
+    plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing {
+        1 = Codappix\SearchCore\DataProcessing\CopyToProcessor
+        1 {
+            to = search_spellcheck
+        }
+
+        2 = Codappix\SearchCore\DataProcessing\CopyToProcessor
+        2 {
+            to = search_all
+        }
+    }
+
+The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards
+all fields, including ``search_spellcheck`` will be copied to ``search_all``.
+E.g. used to index all information into a field for :ref:`spellchecking` or searching with
+different :ref:`mapping`.
+
+The following Processor are available:
+
+.. toctree::
+    :maxdepth: 1
+    :glob:
+
+    dataProcessing/CopyToProcessor
+    dataProcessing/RemoveProcessor
+    dataProcessing/GeoPointProcessor
+
+The following Processor are planned:
+
+    ``Codappix\SearchCore\DataProcessing\ReplaceProcessor``
+        Will execute a search and replace on configured fields.
+
+    ``Codappix\SearchCore\DataProcessing\RootLevelProcessor``
+        Will attach the root level to the record.
+
+    ``Codappix\SearchCore\DataProcessing\ChannelProcessor``
+        Will add a configurable channel to the record, e.g. if you have different areas in your
+        website like "products" and "infos".
+
+    ``Codappix\SearchCore\DataProcessing\RelationResolverProcessor``
+        Resolves all relations using the TCA.
+
+Of course you are able to provide further processors. Just implement
+``Codappix\SearchCore\DataProcessing\ProcessorInterface`` and use the FQCN (=Fully qualified
+class name) as done in the examples above.
+
+By implementing also the same interface as necessary for TYPO3
+:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`, you are able to reuse the same code
+also for Fluid to prepare the same record fetched from DB for your fluid.

Documentation/source/configuration/searching.rst

@@ -0,0 +1,218 @@
+.. _configuration_options_search:
+
+Searching
+=========
+
+.. _size:
+
+size
+----
+
+Used by: Elasticsearch connection while building search query.
+
+Defined how many search results should be fetched to be available in search result.
+
+Example::
+
+    plugin.tx_searchcore.settings.searching.size = 50
+
+Default if not configured is 10.
+
+.. _facets:
+
+facets
+------
+
+Used by: Elasticsearch connection while building search query.
+
+Define aggregations for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-aggregations-bucket-terms-aggregation.html
+Currently only the term facet is provided.
+
+Example::
+
+    plugin.tx_searchcore.settings.searching.facets {
+        contentTypes {
+            field = CType
+        }
+    }
+
+The above example will provide a facet with options for all found ``CType`` results together
+with a count.
+
+.. _filter:
+
+filter
+------
+
+Used by: While building search request.
+
+Define filter that should be set for all requests.
+
+Example::
+
+    plugin.tx_searchcore.settings.searching.filter {
+        property = value
+    }
+
+For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``keyword``.
+
+.. _minimumShouldMatch:
+
+minimumShouldMatch
+------------------
+
+Used by: Elasticsearch connection while building search query.
+
+Define the minimum match for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-minimum-should-match.html
+
+Example::
+
+    plugin.tx_searchcore.settings.searching.minimumShouldMatch = 50%
+
+.. _boost:
+
+boost
+-----
+
+Used by: Elasticsearch connection while building search query.
+
+Define fields that should boost the score for results.
+
+Example::
+
+    plugin.tx_searchcore.settings.searching.boost {
+        search_title = 3
+        search_abstract = 1.5
+    }
+
+For further information take a look at
+https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_boosting_query_clauses.html
+
+.. _fieldValueFactor:
+
+fieldValueFactor
+----------------
+
+Used by: Elasticsearch connection while building search query.
+
+Define a field to use as a factor for scoring. The configuration is passed through to elastic
+search ``field_value_factor``, see: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor
+
+Example::
+
+    plugin.tx_searchcore.settings.searching.field_value_factor {
+        field = rootlineLevel
+        modifier = reciprocal
+        factor = 2
+        missing = 1
+    }
+
+.. _mapping.filter:
+
+mapping.filter
+--------------
+
+Allows to configure filter more in depth. If a filter with the given key exists, the TypoScript will
+be added.
+
+E.g. you submit a filter in form of:
+
+.. code-block:: html
+
+    <f:form.textfield property="filter.distance.location.lat" value="51.168098" />
+    <f:form.textfield property="filter.distance.location.lon" value="6.381384" />
+    <f:form.textfield property="filter.distance.distance" value="100km" />
+
+This will create a ``distance`` filter with subproperties. To make this filter actually work, you
+can add the following TypoScript, which will be added to the filter::
+
+    mapping {
+        filter {
+            distance {
+                field = geo_distance
+                fields {
+                    distance = distance
+                    location = location
+                }
+            }
+        }
+    }
+
+``fields`` has a special meaning here. This will actually map the properties of the filter to fields
+in elasticsearch. In above example they do match, but you can also use different names in your form.
+On the left hand side is the elasticsearch field name, on the right side the one submitted as a
+filter.
+
+The ``field``, in above example ``geo_distance``, will be used as the elasticsearch field for
+filtering. This way you can use arbitrary filter names and map them to existing elasticsearch fields.
+
+.. _fields:
+
+fields
+------
+
+Defines the fields to fetch and search from elasticsearch. With the following sub keys:
+
+``query`` defines the fields to search in. Default is ``_all`` from 5.x times of elasticsearch.
+Configure a comma separated list of fields to search in. This is necessary if you have configured
+special mapping for some fields, or just want to search some fields.
+The most hits get ranked highest. The following is an example configuration::
+
+    fields {
+        query = _all, city
+    }
+
+The following sub properties configure the fields to fetch from elasticsearch:
+
+First ``stored_fields`` which is a list of comma separated fields which actually exist and will be
+added. Typically you will use ``_source`` to fetch the whole indexed fields.
+
+Second is ``script_fields``, which allow you to configure scripted fields for elasticsearch.
+An example might look like the following::
+
+    fields {
+        script_fields {
+            distance {
+                condition = {request.filter.distance.location}
+                script {
+                    params {
+                        lat = {request.filter.distance.location.lat -> f:format.number()}
+                        lon = {request.filter.distance.location.lon -> f:format.number()}
+                    }
+                    lang = painless
+                    inline = doc["location"].arcDistance(params.lat,params.lon) * 0.001
+                }
+            }
+        }
+    }
+
+In above example we add a single ``script_field`` called ``distance``. We add a condition when this
+field should be added. The condition will be parsed as Fluidtemplate and is casted to bool via PHP.
+If the condition is true, or no ``condition`` exists, the ``script_field`` will be added to the
+query. The ``condition`` will be removed and everything else is submitted one to one to
+elasticsearch, except each property is run through Fluidtemplate, to allow you to use information
+from search request, e.g. to insert latitude and longitude from a filter, like in the above example.
+
+.. _sort:
+
+sort
+----
+
+Sort is handled like :ref:`fields`.
+
+.. _mode:
+
+mode
+----
+
+Used by: Controller while preparing action.
+
+Define to switch from search to filter mode.
+
+Example::
+
+    plugin.tx_searchcore.settings.searching {
+        mode = filter
+    }
+
+Only ``filter`` is allowed as value. Will submit an empty query to switch to filter mode.

Documentation/source/features.rst

@@ -15,6 +15,8 @@ configuration needs. Still it's possible to configure the indexer.
 
 Also custom classes can be used as indexers.
 
+Furthermore a finisher for TYPO3 Form-Extension is provided to integrate indexing.
+
 .. _features_search:
 
 Searching
@@ -24,7 +26,7 @@ Currently all fields are searched for a single search input.
 
 Also multiple filter are supported. Filtering results by fields for string contents.
 
-Even facets / aggregates are now possible. Therefore a mapping has to be defined in TypoScript for
+Facets / aggregates are also possible. Therefore a mapping has to be defined in TypoScript for
 indexing, and the facets itself while searching.
 
 .. _features_planned:

Documentation/source/indexer.rst

@@ -21,12 +21,18 @@ further stuff.
 
 The indexer is configurable through the following options:
 
-* :ref:`allowedTables`
-
 * :ref:`rootLineBlacklist`
 
 * :ref:`additionalWhereClause`
 
+* :ref:`abstractFields`
+
+* :ref:`mapping`
+
+* :ref:`index`
+
+* :ref:`dataProcessing`
+
 .. _PagesIndexer:
 
 PagesIndexer
@@ -42,14 +48,18 @@ improve search.
 
 The indexer is configurable through the following options:
 
-* :ref:`allowedTables`
-
 * :ref:`rootLineBlacklist`
 
 * :ref:`additionalWhereClause`
 
 * :ref:`abstractFields`
 
+* :ref:`mapping`
+
+* :ref:`index`
+
+* :ref:`dataProcessing`
+
 .. note::
 
   Not all relations are resolved yet, see :issue:`17` and :pr:`20`.

Documentation/source/usage.rst

@@ -18,6 +18,19 @@ This will index the table ``tt_content`` using the :ref:`TcaIndexer`.
 Only one index per call is available, to run multiple indexers, just make multiple calls.
 The indexers have to be defined in TypoScript via :ref:`configuration_options_index`.
 
+.. _usage_manual_deletion:
+
+Manual deletion
+---------------
+
+You can trigger deletion for a single index from CLI::
+
+    ./typo3/cli_dispatch.phpsh extbase index:delete --identifier 'tt_content'
+
+This will delete the index for the table ``tt_content``.
+
+Only one delete per call is available, to run multiple deletions, just make multiple calls.
+
 .. _usage_auto_indexing:
 
 Auto indexing
@@ -30,6 +43,34 @@ The tables have to be configured via :ref:`configuration_options_index`.
 
   Not all hook operations are supported yet, see :issue:`27`.
 
+.. _usage_form_finisher:
+
+Form finisher
+-------------
+
+A form finisher is provided to integrate indexing into form extension.
+
+Add form finisher to your available finishers and configure it like:
+
+.. code-block:: yaml
+   :linenos:
+
+    -
+        identifier: SearchCoreIndexer
+        options:
+            action: 'delete'
+            indexIdentifier: 'fe_users'
+            recordUid: '{FeUser.user.uid}'
+
+All three options are necessary, where
+
+``action``
+    Is one of ``delete``, ``update`` or ``add``.
+``indexIdentifier``
+    Is a configured index identifier.
+``recordUid``
+    Has to be the uid of the record to index.
+
 .. _usage_searching:
 
 Searching / Frontend Plugin

Tests/Functional/Connection/Elasticsearch/IndexDeletionTest.php

@@ -0,0 +1,50 @@
+<?php
+namespace Codappix\SearchCore\Tests\Functional\Connection\Elasticsearch;
+
+/*
+ * Copyright (C) 2016  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\SearchCore\Domain\Index\IndexerFactory;
+use TYPO3\CMS\Extbase\Object\ObjectManager;
+
+class IndexDeletionTest extends AbstractFunctionalTestCase
+{
+    /**
+     * @test
+     */
+    public function indexIsDeleted()
+    {
+        $this->client->getIndex('typo3content')->create();
+        $this->assertTrue(
+            $this->client->getIndex('typo3content')->exists(),
+            'Could not create index for test.'
+        );
+
+        \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class)
+            ->get(IndexerFactory::class)
+            ->getIndexer('tt_content')
+            ->delete()
+            ;
+
+        $this->assertFalse(
+            $this->client->getIndex('typo3content')->exists(),
+            'Index could not be deleted through command controller.'
+        );
+    }
+}

Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php

@@ -206,6 +206,33 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
         );
     }
 
+    /**
+     * @test
+     */
+    public function indexingDeltedRecordIfRecordShouldBeIndexedButIsNoLongerAvailableAndWasAlreadyIndexed()
+    {
+        \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class)
+            ->get(IndexerFactory::class)
+            ->getIndexer('tt_content')
+            ->indexAllDocuments()
+            ;
+
+        $response = $this->client->request('typo3content/_search?q=*:*');
+        $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.');
+
+        $this->getDatabaseConnection()
+            ->exec_UPDATEquery('tt_content', 'uid = 10', ['hidden' => 1]);
+
+        \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class)
+            ->get(IndexerFactory::class)
+            ->getIndexer('tt_content')
+            ->indexDocument(10)
+            ;
+
+        $response = $this->client->request('typo3content/_search?q=*:*');
+        $this->assertSame($response->getData()['hits']['total'], 1, 'Not exactly 1 document is in index.');
+    }
+
     /**
      * @test
      */

Tests/Functional/Fixtures/BasicSetup.ts

@@ -43,6 +43,10 @@ plugin {
                         field = CType
                     }
                 }
+
+                fields {
+                    query = _all
+                }
             }
         }
     }

Tests/Unit/AbstractUnitTestCase.php

@@ -21,9 +21,40 @@ namespace Codappix\SearchCore\Tests\Unit;
  */
 
 use TYPO3\CMS\Core\Tests\UnitTestCase as CoreTestCase;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Extbase\Object\ObjectManager;
+use TYPO3\CMS\Form\Service\TranslationService;
 
 abstract class AbstractUnitTestCase extends CoreTestCase
 {
+    /**
+     * @var array A backup of registered singleton instances
+     */
+    protected $singletonInstances = [];
+
+    public function setUp()
+    {
+        parent::setUp();
+
+        $this->singletonInstances = GeneralUtility::getSingletonInstances();
+
+        // Disable caching backends to make TYPO3 parts work in unit test mode.
+
+        \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(
+            \TYPO3\CMS\Core\Cache\CacheManager::class
+        )->setCacheConfigurations([
+            'extbase_object' => [
+                'backend' => \TYPO3\CMS\Core\Cache\Backend\NullBackend::class,
+            ],
+        ]);
+    }
+
+    public function tearDown()
+    {
+        GeneralUtility::resetSingletonInstances($this->singletonInstances);
+        parent::tearDown();
+    }
+
     /**
      * @return \TYPO3\CMS\Core\Log\LogManager
      */
@@ -43,4 +74,25 @@ abstract class AbstractUnitTestCase extends CoreTestCase
 
         return $logger;
     }
+
+    /**
+     * Configure translation service mock for Form Finisher.
+     *
+     * This way parseOption will always return the configured value.
+     */
+    protected function configureMockedTranslationService()
+    {
+        $translationService = $this->getMockBuilder(TranslationService::class)->getMock();
+        $translationService->expects($this->any())
+            ->method('translateFinisherOption')
+            ->willReturnCallback(function ($formRuntime, $finisherIdentifier, $optionKey, $optionValue) {
+                return $optionValue;
+            });
+        $objectManager = $this->getMockBuilder(ObjectManager::class)->getMock();
+        $objectManager->expects($this->any())
+            ->method('get')
+            ->with(TranslationService::class)
+            ->willReturn($translationService);
+        GeneralUtility::setSingletonInstance(ObjectManager::class, $objectManager);
+    }
 }

Tests/Unit/Command/IndexCommandControllerTest.php

@@ -91,4 +91,41 @@ class IndexCommandControllerTest extends AbstractUnitTestCase
 
         $this->subject->indexCommand('allowedTable');
     }
+
+    /**
+     * @test
+     */
+    public function deletionIsPossible()
+    {
+        $indexerMock = $this->getMockBuilder(TcaIndexer::class)
+            ->disableOriginalConstructor()
+            ->getMock();
+        $this->subject->expects($this->once())
+            ->method('outputLine')
+            ->with('allowedTable was deleted.');
+        $this->indexerFactory->expects($this->once())
+            ->method('getIndexer')
+            ->with('allowedTable')
+            ->will($this->returnValue($indexerMock));
+
+        $indexerMock->expects($this->once())
+            ->method('delete');
+        $this->subject->deleteCommand('allowedTable');
+    }
+
+    /**
+     * @test
+     */
+    public function deletionForNonExistingIndexerDoesNotWork()
+    {
+        $this->subject->expects($this->once())
+            ->method('outputLine')
+            ->with('No indexer found for: nonAllowedTable');
+        $this->indexerFactory->expects($this->once())
+            ->method('getIndexer')
+            ->with('nonAllowedTable')
+            ->will($this->throwException(new NoMatchingIndexerException));
+
+        $this->subject->deleteCommand('nonAllowedTable');
+    }
 }

Tests/Unit/Configuration/ConfigurationUtilityTest.php

@@ -0,0 +1,143 @@
+<?php
+namespace Codappix\SearchCore\Tests\Unit\Configuration;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\SearchCore\Configuration\ConfigurationUtility;
+use Codappix\SearchCore\Connection\SearchRequestInterface;
+use Codappix\SearchCore\Domain\Model\SearchRequest;
+use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase;
+
+class ConfigurationUtilityTest extends AbstractUnitTestCase
+{
+    /**
+     * @test
+     * @dataProvider possibleRequestAndConfigurationForFluidtemplate
+     */
+    public function recursiveEntriesAreProcessedAsFluidtemplate(SearchRequestInterface $searchRequest, array $array, array $expected)
+    {
+        $subject = new ConfigurationUtility();
+
+        $this->assertSame(
+            $expected,
+            $subject->replaceArrayValuesWithRequestContent($searchRequest, $array),
+            'Entries in array were not parsed as fluid template with search request.'
+        );
+    }
+
+    public function possibleRequestAndConfigurationForFluidtemplate() : array
+    {
+        return [
+            'Nothing in array' => [
+                'searchRequest' => new SearchRequest(),
+                'array' => [],
+                'expected' => [],
+            ],
+            'Small array with nothing to replace' => [
+                'searchRequest' => new SearchRequest(),
+                'array' => [
+                    'key1' => 'value1',
+                ],
+                'expected' => [
+                    'key1' => 'value1',
+                ],
+            ],
+            'Rescursive array with replacements' => [
+                'searchRequest' => call_user_func(function () {
+                    $request = new SearchRequest();
+                    $request->setFilter([
+                        'distance' => [
+                            'location' => '10',
+                        ],
+                    ]);
+                    return $request;
+                }),
+                'array' => [
+                    'sub1' => [
+                        'sub1.1' => '{request.filter.distance.location}',
+                        'sub1.2' => '{request.nonExisting}',
+                    ],
+                ],
+                'expected' => [
+                    'sub1' => [
+                        // Numberics are casted to double
+                        'sub1.1' => 10.0,
+                        'sub1.2' => null,
+                    ],
+                ],
+            ],
+        ];
+    }
+
+    /**
+     * @test
+     * @dataProvider possibleConditionEntries
+     */
+    public function conditionsAreHandledAsExpected(array $entries, array $expected)
+    {
+        $subject = new ConfigurationUtility();
+
+        $this->assertSame(
+            $expected,
+            $subject->filterByCondition($entries),
+            'Conditions were not processed as expected.'
+        );
+    }
+
+    public function possibleConditionEntries() : array
+    {
+        return [
+            'Nothing in array' => [
+                'entries' => [],
+                'expected' => [],
+            ],
+            'Entries without condition' => [
+                'entries' => [
+                    'key1' => 'value1',
+                ],
+                'expected' => [
+                    'key1' => 'value1',
+                ],
+            ],
+            'Entry with matching condition' => [
+                'entries' => [
+                    'sub1' => [
+                        'condition' => true,
+                        'sub1.2' => 'something',
+                    ],
+                ],
+                'expected' => [
+                    'sub1' => [
+                        'sub1.2' => 'something',
+                    ],
+                ],
+            ],
+            'Entry with non matching condition' => [
+                'entries' => [
+                    'sub1' => [
+                        'condition' => false,
+                        'sub1.2' => 'something',
+                    ],
+                ],
+                'expected' => [],
+            ],
+        ];
+    }
+}

Tests/Unit/DataProcessing/CopyToProcessorTest.php

@@ -0,0 +1,85 @@
+<?php
+namespace Codappix\SearchCore\Tests\Unit\DataProcessing;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\SearchCore\DataProcessing\CopyToProcessor;
+use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase;
+
+class CopyToProcessorTest extends AbstractUnitTestCase
+{
+    /**
+     * @test
+     * @dataProvider getPossibleRecordConfigurationCombinations
+     */
+    public function fieldsAreCopiedAsConfigured(array $record, array $configuration, array $expectedRecord)
+    {
+        $subject = new CopyToProcessor();
+        $processedRecord = $subject->processRecord($record, $configuration);
+        $this->assertSame(
+            $expectedRecord,
+            $processedRecord,
+            'The processor did not return the expected processed record.'
+        );
+    }
+
+    /**
+     * @return array
+     */
+    public function getPossibleRecordConfigurationCombinations()
+    {
+        return [
+            'Copy all fields to new field' => [
+                'record' => [
+                    'field 1' => 'Some content like lorem',
+                    'field 2' => 'Some more content like ipsum',
+                ],
+                'configuration' => [
+                    'to' => 'new_field',
+                ],
+                'expectedRecord' => [
+                    'field 1' => 'Some content like lorem',
+                    'field 2' => 'Some more content like ipsum',
+                    'new_field' => 'Some content like lorem' . PHP_EOL . 'Some more content like ipsum',
+                ],
+            ],
+            'Copy all fields with sub array to new field' => [
+                'record' => [
+                    'field 1' => 'Some content like lorem',
+                    'field with sub2' => [
+                        'Tag 1',
+                        'Tag 2',
+                    ],
+                ],
+                'configuration' => [
+                    'to' => 'new_field',
+                ],
+                'expectedRecord' => [
+                    'field 1' => 'Some content like lorem',
+                    'field with sub2' => [
+                        'Tag 1',
+                        'Tag 2',
+                    ],
+                    'new_field' => 'Some content like lorem' . PHP_EOL . 'Tag 1' . PHP_EOL . 'Tag 2',
+                ],
+            ],
+        ];
+    }
+}

Tests/Unit/DataProcessing/GeoPointProcessorTest.php

@@ -0,0 +1,113 @@
+<?php
+namespace Codappix\SearchCore\Tests\Unit\DataProcessing;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\SearchCore\DataProcessing\GeoPointProcessor;
+use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase;
+
+class GeoPointProcessorTest extends AbstractUnitTestCase
+{
+    /**
+     * @test
+     * @dataProvider getPossibleRecordConfigurationCombinations
+     */
+    public function geoPointsAreAddedAsConfigured(array $record, array $configuration, array $expectedRecord)
+    {
+        $subject = new GeoPointProcessor();
+        $processedRecord = $subject->processRecord($record, $configuration);
+        $this->assertSame(
+            $expectedRecord,
+            $processedRecord,
+            'The processor did not return the expected processed record.'
+        );
+    }
+
+    /**
+     * @return array
+     */
+    public function getPossibleRecordConfigurationCombinations()
+    {
+        return [
+            'Create new field with existing lat and lng' => [
+                'record' => [
+                    'lat' => 23.232,
+                    'lng' => 45.43,
+                ],
+                'configuration' => [
+                    'to' => 'location',
+                    'lat' => 'lat',
+                    'lon' => 'lng',
+                ],
+                'expectedRecord' => [
+                    'lat' => 23.232,
+                    'lng' => 45.43,
+                    'location' => [
+                        'lat' => 23.232,
+                        'lon' => 45.43,
+                    ],
+                ],
+            ],
+            'Do not create new field due to missing configuration' => [
+                'record' => [
+                    'lat' => 23.232,
+                    'lng' => 45.43,
+                ],
+                'configuration' => [
+                    'to' => 'location',
+                ],
+                'expectedRecord' => [
+                    'lat' => 23.232,
+                    'lng' => 45.43,
+                ],
+            ],
+            'Do not create new field due to missing lat and lon' => [
+                'record' => [
+                    'lat' => '',
+                    'lng' => '',
+                ],
+                'configuration' => [
+                    'to' => 'location',
+                    'lat' => 'lat',
+                    'lon' => 'lng',
+                ],
+                'expectedRecord' => [
+                    'lat' => '',
+                    'lng' => '',
+                ],
+            ],
+            'Do not create new field due to invalid lat and lon' => [
+                'record' => [
+                    'lat' => 'av',
+                    'lng' => 'dsf',
+                ],
+                'configuration' => [
+                    'to' => 'location',
+                    'lat' => 'lat',
+                    'lon' => 'lng',
+                ],
+                'expectedRecord' => [
+                    'lat' => 'av',
+                    'lng' => 'dsf',
+                ],
+            ],
+        ];
+    }
+}

Tests/Unit/DataProcessing/RemoveProcessorTest.php

@@ -0,0 +1,133 @@
+<?php
+namespace Codappix\SearchCore\Tests\Unit\DataProcessing;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\SearchCore\DataProcessing\RemoveProcessor;
+use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase;
+
+class RemoveProcessorTest extends AbstractUnitTestCase
+{
+    /**
+     * @test
+     * @dataProvider getPossibleRecordConfigurationCombinations
+     */
+    public function fieldsAreCopiedAsConfigured(array $record, array $configuration, array $expectedRecord)
+    {
+        $subject = new RemoveProcessor();
+        $processedRecord = $subject->processRecord($record, $configuration);
+        $this->assertSame(
+            $expectedRecord,
+            $processedRecord,
+            'The processor did not return the expected processed record.'
+        );
+    }
+
+    /**
+     * @return array
+     */
+    public function getPossibleRecordConfigurationCombinations()
+    {
+        return [
+            'Nothing configured' => [
+                'record' => [
+                    'field 1' => 'Some content like lorem',
+                    'field with sub2' => [
+                        'Tag 1',
+                        'Tag 2',
+                    ],
+                ],
+                'configuration' => [
+                ],
+                'expectedRecord' => [
+                    'field 1' => 'Some content like lorem',
+                    'field with sub2' => [
+                        'Tag 1',
+                        'Tag 2',
+                    ],
+                ],
+            ],
+            'Single field configured' => [
+                'record' => [
+                    'field 1' => 'Some content like lorem',
+                    'field with sub2' => [
+                        'Tag 1',
+                        'Tag 2',
+                    ],
+                ],
+                'configuration' => [
+                    'fields' => 'field with sub2',
+                    '_typoScriptNodeValue' => 'Codappix\SearchCore\DataProcessing\RemoveProcessor',
+                ],
+                'expectedRecord' => [
+                    'field 1' => 'Some content like lorem',
+                ],
+            ],
+            'Non existing field configured' => [
+                'record' => [
+                    'field 1' => 'Some content like lorem',
+                    'field with sub2' => [
+                        'Tag 1',
+                        'Tag 2',
+                    ],
+                ],
+                'configuration' => [
+                    'fields' => 'non existing',
+                    '_typoScriptNodeValue' => 'Codappix\SearchCore\DataProcessing\RemoveProcessor',
+                ],
+                'expectedRecord' => [
+                    'field 1' => 'Some content like lorem',
+                    'field with sub2' => [
+                        'Tag 1',
+                        'Tag 2',
+                    ],
+                ],
+            ],
+            'Multiple fields configured' => [
+                'record' => [
+                    'field 1' => 'Some content like lorem',
+                    'field with sub2' => [
+                        'Tag 1',
+                        'Tag 2',
+                    ],
+                    'field 3' => 'Some more like lorem',
+                ],
+                'configuration' => [
+                    'fields' => 'field 3, field with sub2',
+                    '_typoScriptNodeValue' => 'Codappix\SearchCore\DataProcessing\RemoveProcessor',
+                ],
+                'expectedRecord' => [
+                    'field 1' => 'Some content like lorem',
+                ],
+            ],
+            'Fields with "null" san be removed' => [
+                'record' => [
+                    'field 1' => null,
+                ],
+                'configuration' => [
+                    'fields' => 'field 1',
+                    '_typoScriptNodeValue' => 'Codappix\SearchCore\DataProcessing\RemoveProcessor',
+                ],
+                'expectedRecord' => [
+                ],
+            ],
+        ];
+    }
+}

Tests/Unit/Domain/Index/AbstractIndexerTest.php

@@ -0,0 +1,121 @@
+<?php
+namespace Codappix\SearchCore\Tests\Unit\Domain\Index;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
+use Codappix\SearchCore\Configuration\InvalidArgumentException;
+use Codappix\SearchCore\Connection\ConnectionInterface;
+use Codappix\SearchCore\DataProcessing\CopyToProcessor;
+use Codappix\SearchCore\Domain\Index\AbstractIndexer;
+use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase;
+
+class AbstractIndexerTest extends AbstractUnitTestCase
+{
+    /**
+     * @var TcaTableService
+     */
+    protected $subject;
+
+    /**
+     * @var ConfigurationContainerInterface
+     */
+    protected $configuration;
+
+    /**
+     * @var ConnectionInterface
+     */
+    protected $connection;
+
+    public function setUp()
+    {
+        parent::setUp();
+
+        $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock();
+        $this->connection = $this->getMockBuilder(ConnectionInterface::class)->getMock();
+
+        $this->subject = $this->getMockForAbstractClass(AbstractIndexer::class, [
+            $this->connection,
+            $this->configuration
+        ]);
+        $this->subject->injectLogger($this->getMockedLogger());
+        $this->subject->setIdentifier('testTable');
+        $this->subject->expects($this->any())
+            ->method('getDocumentName')
+            ->willReturn('testTable');
+    }
+
+    /**
+     * @test
+     */
+    public function executesConfiguredDataProcessingWithConfiguration()
+    {
+        $record = ['field 1' => 'test'];
+        $expectedRecord = $record;
+        $expectedRecord['new_test_field'] = 'test';
+        $expectedRecord['new_test_field2'] = 'test' . PHP_EOL . 'test';
+        $expectedRecord['search_abstract'] = '';
+
+        $this->configuration->expects($this->exactly(2))
+            ->method('get')
+            ->withConsecutive(['indexing.testTable.dataProcessing'], ['indexing.testTable.abstractFields'])
+            ->will($this->onConsecutiveCalls([
+                '1' => [
+                    '_typoScriptNodeValue' => CopyToProcessor::class,
+                    'to' => 'new_test_field',
+                ],
+                '2' => [
+                    '_typoScriptNodeValue' => CopyToProcessor::class,
+                    'to' => 'new_test_field2',
+                ],
+            ], $this->throwException(new InvalidArgumentException)));
+        $this->subject->expects($this->once())
+            ->method('getRecord')
+            ->with(1)
+            ->willReturn($record)
+            ;
+
+        $this->connection->expects($this->once())->method('addDocument')->with('testTable', $expectedRecord);
+        $this->subject->indexDocument(1);
+    }
+
+    /**
+     * @test
+     */
+    public function executesNoDataProcessingForMissingConfiguration()
+    {
+        $record = ['field 1' => 'test'];
+        $expectedRecord = $record;
+        $expectedRecord['search_abstract'] = '';
+
+        $this->configuration->expects($this->exactly(2))
+            ->method('get')
+            ->withConsecutive(['indexing.testTable.dataProcessing'], ['indexing.testTable.abstractFields'])
+            ->will($this->throwException(new InvalidArgumentException));
+        $this->subject->expects($this->once())
+            ->method('getRecord')
+            ->with(1)
+            ->willReturn($record)
+            ;
+
+        $this->connection->expects($this->once())->method('addDocument')->with('testTable', $expectedRecord);
+        $this->subject->indexDocument(1);
+    }
+}

Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php

@@ -21,6 +21,8 @@ namespace Codappix\SearchCore\Tests\Unit\Domain\Index\TcaIndexer;
  */
 
 use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
+use Codappix\SearchCore\DataProcessing\CopyToProcessor;
+use Codappix\SearchCore\Domain\Index\TcaIndexer\RelationResolver;
 use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService;
 use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase;
 
@@ -82,4 +84,48 @@ class TcaTableServiceTest extends AbstractUnitTestCase
             $this->subject->getWhereClause()
         );
     }
+
+    /**
+     * @test
+     */
+    public function allConfiguredAndAllowedTcaColumnsAreReturnedAsFields()
+    {
+        $GLOBALS['TCA']['test_table'] = [
+            'ctrl' => [
+                'languageField' => 'sys_language',
+            ],
+            'columns' => [
+                'sys_language' => [],
+                't3ver_oid' => [],
+                'available_column' => [
+                    'config' => [
+                        'type' => 'input',
+                    ],
+                ],
+                'user_column' => [
+                    'config' => [
+                        'type' => 'user',
+                    ],
+                ],
+                'passthrough_column' => [
+                    'config' => [
+                        'type' => 'passthrough',
+                    ],
+                ],
+            ],
+        ];
+        $subject = new TcaTableService(
+            'test_table',
+            $this->getMockBuilder(RelationResolver::class)->getMock(),
+            $this->configuration
+        );
+        $this->inject($subject, 'logger', $this->getMockedLogger());
+
+        $this->assertSame(
+            'test_table.uid,test_table.pid,test_table.available_column',
+            $subject->getFields(),
+            ''
+        );
+        unset($GLOBALS['TCA']['test_table']);
+    }
 }

Tests/Unit/Domain/Model/SearchRequestTest.php

@@ -0,0 +1,80 @@
+<?php
+namespace Codappix\SearchCore\Tests\Unit\Domain\Model;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\SearchCore\Domain\Model\SearchRequest;
+use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase;
+
+class SearchRequestTest extends AbstractUnitTestCase
+{
+    /**
+     * @test
+     * @dataProvider possibleEmptyFilter
+     */
+    public function emptyFilterWillNotBeSet(array $filter)
+    {
+        $searchRequest = new SearchRequest();
+        $searchRequest->setFilter($filter);
+
+        $this->assertSame(
+            [],
+            $searchRequest->getFilter(),
+            'Empty filter were set, even if they should not.'
+        );
+    }
+
+    public function possibleEmptyFilter()
+    {
+        return [
+            'Complete empty Filter' => [
+                'filter' => [],
+            ],
+            'Single filter with empty value' => [
+                'filter' => [
+                    'someFilter' => '',
+                ],
+            ],
+            'Single filter with empty recursive values' => [
+                'filter' => [
+                    'someFilter' => [
+                        'someKey' => '',
+                    ],
+                ],
+            ],
+        ];
+    }
+
+    /**
+     * @test
+     */
+    public function filterIsSet()
+    {
+        $filter = ['someField' => 'someValue'];
+        $searchRequest = new SearchRequest();
+        $searchRequest->setFilter($filter);
+
+        $this->assertSame(
+            $filter,
+            $searchRequest->getFilter(),
+            'Filter was not set.'
+        );
+    }
+}

Tests/Unit/Domain/Search/QueryFactoryTest.php

@@ -21,6 +21,7 @@ namespace Codappix\SearchCore\Tests\Unit\Domain\Search;
  */
 
 use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
+use Codappix\SearchCore\Configuration\ConfigurationUtility;
 use Codappix\SearchCore\Configuration\InvalidArgumentException;
 use Codappix\SearchCore\Domain\Model\FacetRequest;
 use Codappix\SearchCore\Domain\Model\SearchRequest;
@@ -44,7 +45,8 @@ class QueryFactoryTest extends AbstractUnitTestCase
         parent::setUp();
 
         $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock();
-        $this->subject = new QueryFactory($this->getMockedLogger(), $this->configuration);
+        $configurationUtility = new ConfigurationUtility();
+        $this->subject = new QueryFactory($this->getMockedLogger(), $this->configuration, $configurationUtility);
     }
 
     /**
@@ -52,10 +54,10 @@ class QueryFactoryTest extends AbstractUnitTestCase
      */
     public function creationOfQueryWorksInGeneral()
     {
-        $this->mockConfiguration();
-
         $searchRequest = new SearchRequest('SearchWord');
 
+        $this->configureConfigurationMockWithDefault();
+
         $query = $this->subject->create($searchRequest);
         $this->assertInstanceOf(
             \Elastica\Query::class,
@@ -69,7 +71,7 @@ class QueryFactoryTest extends AbstractUnitTestCase
      */
     public function filterIsAddedToQuery()
     {
-        $this->mockConfiguration();
+        $this->configureConfigurationMockWithDefault();
 
         $searchRequest = new SearchRequest('SearchWord');
         $searchRequest->setFilter(['field' => 'content']);
@@ -79,7 +81,7 @@ class QueryFactoryTest extends AbstractUnitTestCase
             [
                 ['term' => ['field' => 'content']]
             ],
-            $query->toArray()['query']['function_score']['query']['bool']['filter'],
+            $query->toArray()['query']['bool']['filter'],
             'Filter was not added to query.'
         );
     }
@@ -89,13 +91,11 @@ class QueryFactoryTest extends AbstractUnitTestCase
      */
     public function emptyFilterIsNotAddedToQuery()
     {
-        $this->mockConfiguration();
+        $this->configureConfigurationMockWithDefault();
 
         $searchRequest = new SearchRequest('SearchWord');
         $searchRequest->setFilter([
             'field' => '',
-            'field1' => 0,
-            'field2' => false,
         ]);
 
         $this->assertFalse(
@@ -116,8 +116,7 @@ class QueryFactoryTest extends AbstractUnitTestCase
      */
     public function facetsAreAddedToQuery()
     {
-        $this->mockConfiguration();
+        $this->configureConfigurationMockWithDefault();
-
         $searchRequest = new SearchRequest('SearchWord');
         $searchRequest->addFacet(new FacetRequest('Identifier', 'FieldName'));
         $searchRequest->addFacet(new FacetRequest('Identifier 2', 'FieldName 2'));
@@ -146,8 +145,7 @@ class QueryFactoryTest extends AbstractUnitTestCase
      */
     public function sizeIsAddedToQuery()
     {
-        $this->mockConfiguration();
+        $this->configureConfigurationMockWithDefault();
-
         $searchRequest = new SearchRequest('SearchWord');
         $searchRequest->setLimit(45);
         $searchRequest->setOffset(35);
@@ -170,9 +168,8 @@ class QueryFactoryTest extends AbstractUnitTestCase
      */
     public function searchTermIsAddedToQuery()
     {
-        $this->mockConfiguration();
-
         $searchRequest = new SearchRequest('SearchWord');
+        $this->configureConfigurationMockWithDefault();
         $query = $this->subject->create($searchRequest);
 
         $this->assertSame(
@@ -191,7 +188,7 @@ class QueryFactoryTest extends AbstractUnitTestCase
                     ],
                 ],
             ],
-            $query->toArray()['query']['function_score']['query'],
+            $query->toArray()['query'],
             'Search term was not added to query as expected.'
         );
     }
@@ -201,28 +198,38 @@ class QueryFactoryTest extends AbstractUnitTestCase
      */
     public function minimumShouldMatchIsAddedToQuery()
     {
-        $this->configuration->expects($this->once())
-            ->method('getIfExists')
-            ->with('searching.minimumShouldMatch')
-            ->willReturn('50%');
-        $this->mockConfiguration();
-
         $searchRequest = new SearchRequest('SearchWord');
+        $this->configuration->expects($this->any())
+            ->method('getIfExists')
+            ->withConsecutive(
+                ['searching.minimumShouldMatch'],
+                ['searching.sort']
+            )
+            ->will($this->onConsecutiveCalls(
+                '50%',
+                null
+            ));
+        $this->configureConfigurationMockWithDefault();
         $query = $this->subject->create($searchRequest);
 
-        $this->assertArraySubset(
+        $this->assertSame(
             [
                 'bool' => [
                     'must' => [
                         [
                             'multi_match' => [
+                                'type' => 'most_fields',
+                                'query' => 'SearchWord',
+                                'fields' => [
+                                    'test_field',
+                                ],
                                 'minimum_should_match' => '50%',
                             ],
                         ],
                     ],
                 ],
             ],
-            $query->toArray()['query']['function_score']['query'],
+            $query->toArray()['query'],
             'minimum_should_match was not added to query as configured.'
         );
     }
@@ -234,15 +241,23 @@ class QueryFactoryTest extends AbstractUnitTestCase
     {
         $searchRequest = new SearchRequest('SearchWord');
 
-        $this->configuration->expects($this->exactly(3))
+        $this->configuration->expects($this->any())
             ->method('get')
-            ->withConsecutive(['searching.fields'], ['searching.boost'], ['searching.fieldValueFactor'])
+            ->withConsecutive(
+                ['searching.fields.query'],
+                ['searching.boost'],
+                ['searching.fields.stored_fields'],
+                ['searching.fields.script_fields'],
+                ['searching.fieldValueFactor']
+            )
             ->will($this->onConsecutiveCalls(
                 'test_field',
                 [
                     'search_title' => 3,
                     'search_abstract' => 1.5,
                 ],
+                $this->throwException(new InvalidArgumentException),
+                $this->throwException(new InvalidArgumentException),
                 $this->throwException(new InvalidArgumentException)
             ));
 
@@ -283,12 +298,20 @@ class QueryFactoryTest extends AbstractUnitTestCase
             'factor' => '2',
             'missing' => '1',
         ];
-        $this->configuration->expects($this->exactly(3))
+        $this->configuration->expects($this->any())
             ->method('get')
-            ->withConsecutive(['searching.fields'], ['searching.boost'], ['searching.fieldValueFactor'])
+            ->withConsecutive(
+                ['searching.fields.query'],
+                ['searching.boost'],
+                ['searching.fields.stored_fields'],
+                ['searching.fields.script_fields'],
+                ['searching.fieldValueFactor']
+            )
             ->will($this->onConsecutiveCalls(
                 'test_field',
                 $this->throwException(new InvalidArgumentException),
+                $this->throwException(new InvalidArgumentException),
+                $this->throwException(new InvalidArgumentException),
                 $fieldConfig
             ));
 
@@ -324,27 +347,279 @@ class QueryFactoryTest extends AbstractUnitTestCase
      */
     public function emptySearchStringWillNotAddSearchToQuery()
     {
-        $this->mockConfiguration();
-
         $searchRequest = new SearchRequest();
 
+        $this->configureConfigurationMockWithDefault();
+
         $query = $this->subject->create($searchRequest);
-        $this->assertNull(
+        $this->assertInstanceOf(
-            $query->toArray()['query']['function_score']['query'],
+            stdClass,
+            $query->toArray()['query']['match_all'],
             'Empty search request does not create expected query.'
         );
     }
 
-    protected function mockConfiguration()
+    /**
+     * @test
+     */
+    public function configuredQueryFieldsAreAddedToQuery()
+    {
+        $searchRequest = new SearchRequest('SearchWord');
+
+        $this->configuration->expects($this->any())
+            ->method('get')
+            ->withConsecutive(
+                ['searching.fields.query'],
+                ['searching.boost'],
+                ['searching.fields.stored_fields'],
+                ['searching.fields.script_fields'],
+                ['searching.fieldValueFactor']
+            )
+            ->will($this->onConsecutiveCalls(
+                'test_field, field1, field2',
+                $this->throwException(new InvalidArgumentException),
+                $this->throwException(new InvalidArgumentException),
+                $this->throwException(new InvalidArgumentException),
+                $this->throwException(new InvalidArgumentException)
+            ));
+
+        $query = $this->subject->create($searchRequest);
+        $this->assertArraySubset(
+            [
+                'bool' => [
+                    'must' => [
+                        [
+                            'multi_match' => [
+                                'type' => 'most_fields',
+                                'query' => 'SearchWord',
+                                'fields' => [
+                                    'test_field',
+                                    'field1',
+                                    'field2',
+                                ],
+                            ],
+                        ],
+                    ],
+                ],
+            ],
+            $query->toArray()['query'],
+            'Configured fields were not added to query as configured.'
+        );
+    }
+
+    /**
+     * @test
+     */
+    public function storedFieldsAreAddedToQuery()
+    {
+        $searchRequest = new SearchRequest();
+
+        $this->configuration->expects($this->any())
+            ->method('get')
+            ->withConsecutive(
+                ['searching.boost'],
+                ['searching.fields.stored_fields'],
+                ['searching.fields.script_fields'],
+                ['searching.fieldValueFactor']
+            )
+            ->will($this->onConsecutiveCalls(
+                $this->throwException(new InvalidArgumentException),
+                '_source, something,nothing',
+                $this->throwException(new InvalidArgumentException),
+                $this->throwException(new InvalidArgumentException)
+            ));
+
+        $query = $this->subject->create($searchRequest);
+        $this->assertSame(
+            ['_source', 'something', 'nothing'],
+            $query->toArray()['stored_fields'],
+            'Stored fields were not added to query as expected.'
+        );
+    }
+
+    /**
+     * @test
+     */
+    public function storedFieldsAreNotAddedToQuery()
+    {
+        $searchRequest = new SearchRequest();
+
+        $this->configuration->expects($this->any())
+            ->method('get')
+            ->withConsecutive(
+                ['searching.boost'],
+                ['searching.fields.stored_fields'],
+                ['searching.fields.script_fields'],
+                ['searching.fieldValueFactor']
+            )
+            ->will($this->onConsecutiveCalls(
+                $this->throwException(new InvalidArgumentException),
+                $this->throwException(new InvalidArgumentException),
+                $this->throwException(new InvalidArgumentException),
+                $this->throwException(new InvalidArgumentException)
+            ));
+
+        $query = $this->subject->create($searchRequest);
+        $this->assertFalse(
+            isset($query->toArray()['stored_fields']),
+            'Stored fields were added to query even if not configured.'
+        );
+    }
+
+    /**
+     * @test
+     */
+    public function scriptFieldsAreAddedToQuery()
+    {
+        $searchRequest = new SearchRequest('query value');
+
+        $this->configuration->expects($this->any())
+            ->method('get')
+            ->withConsecutive(
+                ['searching.fields.query'],
+                ['searching.boost'],
+                ['searching.fields.stored_fields'],
+                ['searching.fields.script_fields'],
+                ['searching.fieldValueFactor']
+            )
+            ->will($this->onConsecutiveCalls(
+                'test_field',
+                $this->throwException(new InvalidArgumentException),
+                $this->throwException(new InvalidArgumentException),
+                [
+                    'field1' => [
+                        'config' => 'something',
+                    ],
+                    'field2' => [
+                        'config' => '{request.query}',
+                    ],
+                ],
+                $this->throwException(new InvalidArgumentException)
+            ));
+
+        $query = $this->subject->create($searchRequest);
+        $this->assertSame(
+            [
+                'field1' => [
+                    'config' => 'something',
+                ],
+                'field2' => [
+                    'config' => 'query value',
+                ],
+            ],
+            $query->toArray()['script_fields'],
+            'Script fields were not added to query as expected.'
+        );
+    }
+
+    /**
+     * @test
+     */
+    public function scriptFieldsAreNotAddedToQuery()
+    {
+        $searchRequest = new SearchRequest();
+
+        $this->configuration->expects($this->any())
+            ->method('get')
+            ->withConsecutive(
+                ['searching.boost'],
+                ['searching.fields.stored_fields'],
+                ['searching.fields.script_fields'],
+                ['searching.fieldValueFactor']
+            )
+            ->will($this->onConsecutiveCalls(
+                $this->throwException(new InvalidArgumentException),
+                $this->throwException(new InvalidArgumentException),
+                $this->throwException(new InvalidArgumentException),
+                $this->throwException(new InvalidArgumentException)
+            ));
+
+        $query = $this->subject->create($searchRequest);
+        $this->assertTrue(
+            !isset($query->toArray()['script_fields']),
+            'Script fields were added to query even if not configured.'
+        );
+    }
+
+    /**
+     * @test
+     */
+    public function sortIsAddedToQuery()
+    {
+        $searchRequest = new SearchRequest('query value');
+
+        $this->configuration->expects($this->any())
+            ->method('getIfExists')
+            ->withConsecutive(
+                ['searching.minimumShouldMatch'],
+                ['searching.sort']
+            )
+            ->will($this->onConsecutiveCalls(
+                null,
+                [
+                    'field1' => [
+                        'config' => 'something',
+                    ],
+                    'field2' => [
+                        'config' => '{request.query}',
+                    ],
+                ]
+            ));
+
+        $this->configureConfigurationMockWithDefault();
+
+        $query = $this->subject->create($searchRequest);
+        $this->assertSame(
+            [
+                'field1' => [
+                    'config' => 'something',
+                ],
+                'field2' => [
+                    'config' => 'query value',
+                ],
+            ],
+            $query->toArray()['sort'],
+            'Sort was not added to query as expected.'
+        );
+    }
+
+    /**
+     * @test
+     */
+    public function sortIsNotAddedToQuery()
+    {
+        $searchRequest = new SearchRequest('query value');
+
+        $this->configuration->expects($this->any())
+            ->method('getIfExists')
+            ->withConsecutive(
+                ['searching.minimumShouldMatch'],
+                ['searching.sort']
+            )
+            ->will($this->onConsecutiveCalls(
+                null,
+                null
+            ));
+
+        $this->configureConfigurationMockWithDefault();
+
+        $query = $this->subject->create($searchRequest);
+        $this->assertTrue(
+            !isset($query->toArray()['sort']),
+            'Sort was added to query even if not configured.'
+        );
+    }
+
+    protected function configureConfigurationMockWithDefault()
     {
         $this->configuration->expects($this->any())
             ->method('get')
-            ->will($this->returnCallback(function ($option) {
+            ->will($this->returnCallback(function ($configName) {
-                if ($option === 'searching.fields') {
+                if ($configName === 'searching.fields.query') {
                     return 'test_field';
                 }
 
-                return $this->throwException(new InvalidArgumentException);
+                throw new InvalidArgumentException();
             }));
     }
 }

Tests/Unit/Domain/Search/SearchServiceTest.php

@@ -178,4 +178,29 @@ class SearchServiceTest extends AbstractUnitTestCase
         $searchRequest->setFilter(['anotherProperty' => 'anything']);
         $this->subject->search($searchRequest);
     }
+
+    /**
+     * @test
+     */
+    public function emptyConfiguredFilterIsNotChangingRequestWithExistingFilter()
+    {
+        $this->configuration->expects($this->exactly(2))
+            ->method('getIfExists')
+            ->withConsecutive(['searching.size'], ['searching.facets'])
+            ->will($this->onConsecutiveCalls(null, null));
+        $this->configuration->expects($this->exactly(1))
+            ->method('get')
+            ->with('searching.filter')
+            ->willReturn(['anotherProperty' => '']);
+
+        $this->connection->expects($this->once())
+            ->method('search')
+            ->with($this->callback(function ($searchRequest) {
+                return $searchRequest->getFilter() === ['anotherProperty' => 'anything'];
+            }));
+
+        $searchRequest = new SearchRequest('SearchWord');
+        $searchRequest->setFilter(['anotherProperty' => 'anything']);
+        $this->subject->search($searchRequest);
+    }
 }

composer.json

@@ -16,7 +16,7 @@
         }
     },
     "require" : {
-        "php": ">=5.6.0",
+        "php": ">=7.0",
         "typo3/cms": "~7.6",
         "ruflin/elastica": "~3.2"
     },
@@ -35,6 +35,9 @@
         ]
     },
     "extra": {
+        "branch-alias": {
+            "dev-develop": "1.0.x-dev"
+        },
         "typo3/cms": {
             "cms-package-dir": "{$vendor-dir}/typo3/cms",
             "web-dir": ".Build/web"

ext_emconf.php

@@ -8,7 +8,7 @@ $EM_CONF[$_EXTKEY] = [
     'constraints' => [
         'depends' => [
             'typo3' => '7.6.0-7.6.99',
-            'php' => '5.6.0-7.99.99'
+            'php' => '7.0.0-7.99.99'
         ],
         'conflicts' => [],
     ],

ext_localconf.php

@@ -33,7 +33,7 @@ call_user_func(
                 'Search' => 'search'
             ],
             [
-                'Search' => 'search' // TODO: Enable caching. But submitting form results in previous result?!
+                'Search' => 'search'
             ]
         );