Merge pull request #137 from Codappix/release/0.0.1

Release: 0.0.1

.phan/config.php

@@ -0,0 +1,96 @@
+<?php
+
+/**
+ * This configuration will be read and overlaid on top of the
+ * default configuration. Command line arguments will be applied
+ * after this file is read.
+ */
+return [
+
+    // Supported values: '7.0', '7.1', '7.2', null.
+    // If this is set to null,
+    // then Phan assumes the PHP version which is closest to the minor version
+    // of the php executable used to execute phan.
+    "target_php_version" => '7.0',
+
+    // Override to hardcode existence and types of (non-builtin) globals.
+    // Class names should be prefixed with '\\'.
+    // (E.g. ['_FOO' => '\\FooClass', 'page' => '\\PageClass', 'userId' => 'int'])
+    'globals_type_map' => [
+        '_EXTKEY' => 'string',
+        'EM_CONF' => 'array',
+    ],
+
+    // A list of directories that should be parsed for class and
+    // method information. After excluding the directories
+    // defined in exclude_analysis_directory_list, the remaining
+    // files will be statically analyzed for errors.
+    //
+    // Thus, both first-party and third-party code being used by
+    // your application should be included in this list.
+    'directory_list' => [
+        'Classes',
+        '.Build/vendor',
+    ],
+
+    // A list of files to include in analysis
+    'file_list' => [
+        'ext_emconf.php',
+        'ext_tables.php',
+        'ext_localconf.php',
+    ],
+
+    // A directory list that defines files that will be excluded
+    // from static analysis, but whose class and method
+    // information should be included.
+    //
+    // Generally, you'll want to include the directories for
+    // third-party code (such as "vendor/") in this list.
+    //
+    // n.b.: If you'd like to parse but not analyze 3rd
+    //       party code, directories containing that code
+    //       should be added to the `directory_list` as
+    //       to `exclude_analysis_directory_list`.
+    "exclude_analysis_directory_list" => [
+        '.Build/vendor'
+    ],
+
+    // A list of directories that should be parsed for class and
+    // method information. After excluding the directories
+    // defined in exclude_analysis_directory_list, the remaining
+    // files will be statically analyzed for errors.
+    //
+    // Thus, both first-party and third-party code being used by
+    // your application should be included in this list.
+    'directory_list' => [
+        'Classes',
+        // 'Tests',
+        '.Build/vendor',
+    ],
+
+    // The number of processes to fork off during the analysis phase.
+    'processes' => 3,
+
+    // Add any issue types (such as 'PhanUndeclaredMethod')
+    // here to inhibit them from being reported
+    'suppress_issue_types' => [
+        'PhanDeprecatedFunction', // For now
+        'PhanParamTooMany', // For now, due to ObjectManager->get()
+    ],
+
+    // A list of plugin files to execute.
+    // See https://github.com/phan/phan/tree/master/.phan/plugins for even more.
+    // (Pass these in as relative paths.
+    // The 0.10.2 release will allow passing 'AlwaysReturnPlugin' if referring to a plugin that is bundled with Phan)
+    'plugins' => [
+        // checks if a function, closure or method unconditionally returns.
+        'AlwaysReturnPlugin',  // can also be written as 'vendor/phan/phan/.phan/plugins/AlwaysReturnPlugin.php'
+        // Checks for syntactically unreachable statements in
+        // the global scope or function bodies.
+        'UnreachableCodePlugin',
+        'DollarDollarPlugin',
+        'DuplicateArrayKeyPlugin',
+        'PregRegexCheckerPlugin',
+        'PrintfCheckerPlugin',
+    ],
+];

.scrutinizer.yml

@@ -1,3 +1,11 @@
+build:
+    nodes:
+        analysis:
+            project_setup:
+                override: true
+            tests:
+                override: [php-scrutinizer-run]
+
 filter:
     excluded_paths:
         - 'Configuration/*'
@@ -19,7 +27,7 @@ tools:
     php_hhvm:
         enabled: true
         config:
-            use_undeclared_constant:  false
+            use_undeclared_constant: false
 
     php_mess_detector:
         enabled: true
@@ -34,5 +42,5 @@ tools:
         enabled: true
     # We generate code coverage during tests at travis and will send them here
     external_code_coverage:
-      runs: 2
+        runs: 2
-      timeout: 1200
+        timeout: 1200

.travis.yml

@@ -1,9 +1,19 @@
+sudo: true
+
+addons:
+  apt:
+    packages:
+      - oracle-java8-set-default
+before_install:
+  - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.2.0.deb && sudo dpkg -i --force-confnew elasticsearch-5.2.0.deb && sudo service elasticsearch start
+  - mysql -u root -e 'GRANT ALL ON `typo3_ci_ft%`.* TO travis@127.0.0.1;'
+
 language: php
 
 php:
-  - 5.6
   - 7.0
   - 7.1
+  - 7.2
 
 env:
   global:
@@ -17,34 +27,20 @@ env:
     - typo3DatabasePassword=""
   matrix:
     - TYPO3_VERSION="~7.6"
-    - TYPO3_VERSION="~8"
+    - TYPO3_VERSION="~8.7"
-    - TYPO3_VERSION="dev-master"
 
 matrix:
   fast_finish: true
-  exclude:
-    # TYPO3 no longer supports 5.6
-    - env: TYPO3_VERSION="~8"
-      php: 5.6
-    - env: TYPO3_VERSION="dev-master"
-      php: 5.6
-  allow_failures:
-    - env: TYPO3_VERSION="~8"
-      php: 7.0
-    - env: TYPO3_VERSION="~8"
-      php: 7.1
-    - env: TYPO3_VERSION="dev-master"
-      php: 7.0
-    - env: TYPO3_VERSION="dev-master"
-      php: 7.1
 
 services:
   - mysql
-  - elasticsearch
 
 install: make install
 
-script: make functionalTests
+script:
+  - make cgl
+  - make unitTests
+  - make functionalTests
 
 after_script:
   - make uploadCodeCoverage

Classes/Command/IndexCommandController.php

@@ -21,7 +21,7 @@ namespace Codappix\SearchCore\Command;
  */
 
 use Codappix\SearchCore\Domain\Index\IndexerFactory;
-use TYPO3\CMS\Core\Utility\GeneralUtility;
+use Codappix\SearchCore\Domain\Index\NoMatchingIndexerException;
 use TYPO3\CMS\Extbase\Mvc\Controller\CommandController;
 
 /**
@@ -34,12 +34,6 @@ class IndexCommandController extends CommandController
      */
     protected $indexerFactory;
 
-    /**
-     * @var \Codappix\SearchCore\Configuration\ConfigurationContainerInterface
-     * @inject
-     */
-    protected $configuration;
-
     /**
      * @param IndexerFactory $factory
      */
@@ -49,19 +43,32 @@ class IndexCommandController extends CommandController
     }
 
     /**
-     * Will index the given table or everything.
+     * Will index the given identifier.
      *
-     * @param string $table
+     * @param string $identifier
      */
-    public function indexCommand($table)
+    public function indexCommand(string $identifier)
     {
-        // TODO: Allow to index multiple tables at once?
+        try {
-        // TODO: Also allow to index everything?
+            $this->indexerFactory->getIndexer($identifier)->indexAllDocuments();
-        if (! in_array($table, GeneralUtility::trimExplode(',', $this->configuration->get('indexer.tca.allowedTables')))) {
+            $this->outputLine($identifier . ' was indexed.');
-            $this->outputLine('Table is not allowed for indexing.');
+        } catch (NoMatchingIndexerException $e) {
-            $this->quit(1);
+            $this->outputLine('No indexer found for: ' . $identifier);
+        }
+    }
+
+    /**
+     * Will delete the given identifier.
+     *
+     * @param string $identifier
+     */
+    public function deleteCommand(string $identifier)
+    {
+        try {
+            $this->indexerFactory->getIndexer($identifier)->delete();
+            $this->outputLine($identifier . ' was deleted.');
+        } catch (NoMatchingIndexerException $e) {
+            $this->outputLine('No indexer found for: ' . $identifier);
         }
-        $this->indexerFactory->getIndexer($table)->indexAllDocuments();
-        $this->outputLine('Table was indexed.');
     }
 }

Classes/Compatibility/ImplementationRegistrationService.php

@@ -0,0 +1,56 @@
+<?php
+namespace Codappix\SearchCore\Compatibility;
+
+/*
+ * Copyright (C) 2018  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Core\Utility\VersionNumberUtility;
+use TYPO3\CMS\Extbase\Object\Container\Container;
+
+/**
+ * Register different concrete implementations, depending on current TYPO3 version.
+ * This way we can provide working implementations for multiple TYPO3 versions.
+ */
+class ImplementationRegistrationService
+{
+    public static function registerImplementations()
+    {
+        $container = GeneralUtility::makeInstance(Container::class);
+        if (VersionNumberUtility::convertVersionNumberToInteger(TYPO3_version) >= 8000000) {
+            $container->registerImplementation(
+                \Codappix\SearchCore\Compatibility\TypoScriptServiceInterface::class,
+                \Codappix\SearchCore\Compatibility\TypoScriptService::class
+            );
+            $container->registerImplementation(
+                \Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface::class,
+                \Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService::class
+            );
+        } else {
+            $container->registerImplementation(
+                \Codappix\SearchCore\Compatibility\TypoScriptServiceInterface::class,
+                \Codappix\SearchCore\Compatibility\TypoScriptService76::class
+            );
+            $container->registerImplementation(
+                \Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface::class,
+                \Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService76::class
+            );
+        }
+    }
+}

Classes/Compatibility/TypoScriptService.php

@@ -0,0 +1,31 @@
+<?php
+namespace Codappix\SearchCore\Compatibility;
+
+/*
+ * Copyright (C) 2018  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use TYPO3\CMS\Core\TypoScript\TypoScriptService as CoreTypoScriptService;
+
+/**
+ * Used since TYPO3 CMS 8.7.
+ */
+class TypoScriptService extends CoreTypoScriptService implements TypoScriptServiceInterface
+{
+
+}

Classes/Compatibility/TypoScriptService76.php

@@ -0,0 +1,31 @@
+<?php
+namespace Codappix\SearchCore\Compatibility;
+
+/*
+ * Copyright (C) 2018  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use TYPO3\CMS\Extbase\Service\TypoScriptService as CoreTypoScriptService;
+
+/**
+ * Used before TYPO3 CMS 8.7.
+ */
+class TypoScriptService76 extends CoreTypoScriptService implements TypoScriptServiceInterface
+{
+
+}

Classes/Compatibility/TypoScriptServiceInterface.php

@@ -0,0 +1,30 @@
+<?php
+namespace Codappix\SearchCore\Compatibility;
+
+/*
+ * Copyright (C) 2018  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * Allows to use DI configuration to switch concrete implementation, depending
+ * on current TYPO3 Version.
+ */
+interface TypoScriptServiceInterface
+{
+    public function convertPlainArrayToTypoScriptArray(array $plainArray);
+}

Classes/Configuration/ConfigurationContainer.php

@@ -39,7 +39,6 @@ class ConfigurationContainer implements ConfigurationContainerInterface
     /**
      * Inject settings via ConfigurationManager.
      *
-     * @param ConfigurationManagerInterface $configurationManager
      * @throws NoConfigurationException
      */
     public function injectConfigurationManager(ConfigurationManagerInterface $configurationManager)
@@ -59,7 +58,7 @@ class ConfigurationContainer implements ConfigurationContainerInterface
      * @return mixed
      * @throws InvalidArgumentException
      */
-    public function get($path)
+    public function get(string $path)
     {
         $value = ArrayUtility::getValueByPath($this->settings, $path);
 
@@ -77,7 +76,7 @@ class ConfigurationContainer implements ConfigurationContainerInterface
      * @param string $path In dot notation.
      * @return mixed
      */
-    public function getIfExists($path)
+    public function getIfExists(string $path)
     {
         return ArrayUtility::getValueByPath($this->settings, $path);
     }

Classes/Configuration/ConfigurationContainerInterface.php

@@ -34,8 +34,10 @@ interface ConfigurationContainerInterface extends Singleton
      *
      * @param string $path In dot notation. E.g. indexer.tca.allowedTables
      * @return mixed
+     *
+     * @throws InvalidArgumentException
      */
-    public function get($path);
+    public function get(string $path);
 
     /**
      * Same as get but will not throw an exception but return null.
@@ -43,5 +45,5 @@ interface ConfigurationContainerInterface extends Singleton
      * @param string $path In dot notation.
      * @return mixed|null
      */
-    public function getIfExists($path);
+    public function getIfExists(string $path);
 }

Classes/Configuration/ConfigurationUtility.php

@@ -0,0 +1,69 @@
+<?php
+namespace Codappix\SearchCore\Configuration;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\SearchCore\Connection\SearchRequestInterface;
+use TYPO3\CMS\Fluid\View\StandaloneView;
+
+class ConfigurationUtility
+{
+    /**
+     * Will parse all entries, recursive as fluid template, with request variable set to $searchRequest.
+     */
+    public function replaceArrayValuesWithRequestContent(SearchRequestInterface $searchRequest, array $array) : array
+    {
+        array_walk_recursive($array, function (&$value, $key, SearchRequestInterface $searchRequest) {
+            $template = new StandaloneView();
+            $template->assign('request', $searchRequest);
+            $template->setTemplateSource($value);
+            $value = $template->render();
+
+            // As elasticsearch does need some doubles to be send as doubles.
+            if (is_numeric($value)) {
+                $value = (float) $value;
+            }
+        }, $searchRequest);
+
+        return $array;
+    }
+
+    /**
+     * Will check all entries, whether they have a condition and filter entries out, where condition is false.
+     * Also will remove condition in the end.
+     */
+    public function filterByCondition(array $entries) : array
+    {
+        $entries = array_filter($entries, function ($entry) {
+            return !is_array($entry)
+                || !array_key_exists('condition', $entry)
+                || (bool) $entry['condition'] === true
+                ;
+        });
+
+        foreach ($entries as $key => $entry) {
+            if (is_array($entry) && array_key_exists('condition', $entry)) {
+                unset($entries[$key]['condition']);
+            }
+        }
+
+        return $entries;
+    }
+}

Classes/Connection/ConnectionInterface.php

@@ -28,53 +28,44 @@ interface ConnectionInterface
     /**
      * Will add a new document.
      *
-     * @param string $documentType
-     * @param array $document
-     *
      * @return void
      */
-    public function addDocument($documentType, array $document);
+    public function addDocument(string $documentType, array $document);
 
     /**
      * Add the given documents.
      *
-     * @param string $documentType
-     * @param array $documents
-     *
      * @return void
      */
-    public function addDocuments($documentType, array $documents);
+    public function addDocuments(string $documentType, array $documents);
 
     /**
      * Will update an existing document.
      *
      * NOTE: Batch updating is not yet supported.
      *
-     * @param string $documentType
-     * @param array $document
-     *
      * @return void
      */
-    public function updateDocument($documentType, array $document);
+    public function updateDocument(string $documentType, array $document);
 
     /**
      * Will remove an existing document.
      *
      * NOTE: Batch deleting is not yet supported.
      *
-     * @param string $documentType
-     * @param int $identifier
-     *
      * @return void
      */
-    public function deleteDocument($documentType, $identifier);
+    public function deleteDocument(string $documentType, string $identifier);
 
     /**
      * Search by given request and return result.
-     *
-     * @param SearchRequestInterface $searchRequest
-     *
-     * @return SearchResultInterface
      */
-    public function search(SearchRequestInterface $searchRequest);
+    public function search(SearchRequestInterface $searchRequest) : SearchResultInterface;
+
+    /**
+     * Will delete the whole index / db.
+     *
+     * @return void
+     */
+    public function deleteIndex(string $documentType);
 }

Classes/Connection/Elasticsearch.php

@@ -20,7 +20,10 @@ namespace Codappix\SearchCore\Connection;
  * 02110-1301, USA.
  */
 
+use Codappix\SearchCore\Connection\Elasticsearch\SearchResult;
+use Codappix\SearchCore\Domain\Search\QueryFactory;
 use TYPO3\CMS\Core\SingletonInterface as Singleton;
+use TYPO3\CMS\Extbase\Object\ObjectManagerInterface;
 
 /**
  * Outer wrapper to elasticsearch.
@@ -42,16 +45,31 @@ class Elasticsearch implements Singleton, ConnectionInterface
      */
     protected $typeFactory;
 
+    /**
+     * @var Elasticsearch\MappingFactory
+     */
+    protected $mappingFactory;
+
     /**
      * @var Elasticsearch\DocumentFactory
      */
     protected $documentFactory;
 
+    /**
+     * @var QueryFactory
+     */
+    protected $queryFactory;
+
     /**
      * @var \TYPO3\CMS\Core\Log\Logger
      */
     protected $logger;
 
+    /**
+     * @var ObjectManagerInterface
+     */
+    protected $objectManager;
+
     /**
      * Inject log manager to get concrete logger from it.
      *
@@ -62,25 +80,39 @@ class Elasticsearch implements Singleton, ConnectionInterface
         $this->logger = $logManager->getLogger(__CLASS__);
     }
 
+    /**
+     * @param ObjectManagerInterface $objectManager
+     */
+    public function injectObjectManager(ObjectManagerInterface $objectManager)
+    {
+        $this->objectManager = $objectManager;
+    }
+
     /**
      * @param Elasticsearch\Connection $connection
      * @param Elasticsearch\IndexFactory $indexFactory
      * @param Elasticsearch\TypeFactory $typeFactory
+     * @param Elasticsearch\MappingFactory $mappingFactory
      * @param Elasticsearch\DocumentFactory $documentFactory
+     * @param QueryFactory $queryFactory
      */
     public function __construct(
         Elasticsearch\Connection $connection,
         Elasticsearch\IndexFactory $indexFactory,
         Elasticsearch\TypeFactory $typeFactory,
-        Elasticsearch\DocumentFactory $documentFactory
+        Elasticsearch\MappingFactory $mappingFactory,
+        Elasticsearch\DocumentFactory $documentFactory,
+        QueryFactory $queryFactory
     ) {
         $this->connection = $connection;
         $this->indexFactory = $indexFactory;
         $this->typeFactory = $typeFactory;
+        $this->mappingFactory = $mappingFactory;
         $this->documentFactory = $documentFactory;
+        $this->queryFactory = $queryFactory;
     }
 
-    public function addDocument($documentType, array $document)
+    public function addDocument(string $documentType, array $document)
     {
         $this->withType(
             $documentType,
@@ -90,7 +122,7 @@ class Elasticsearch implements Singleton, ConnectionInterface
         );
     }
 
-    public function deleteDocument($documentType, $identifier)
+    public function deleteDocument(string $documentType, string $identifier)
     {
         try {
             $this->withType(
@@ -100,11 +132,14 @@ class Elasticsearch implements Singleton, ConnectionInterface
                 }
             );
         } catch (\Elastica\Exception\NotFoundException $exception) {
-            $this->logger->debug('Tried to delete document in index, which does not exist.', [$documentType, $identifier]);
+            $this->logger->debug(
+                'Tried to delete document in index, which does not exist.',
+                [$documentType, $identifier]
+            );
         }
     }
 
-    public function updateDocument($documentType, array $document)
+    public function updateDocument(string $documentType, array $document)
     {
         $this->withType(
             $documentType,
@@ -114,7 +149,7 @@ class Elasticsearch implements Singleton, ConnectionInterface
         );
     }
 
-    public function addDocuments($documentType, array $documents)
+    public function addDocuments(string $documentType, array $documents)
     {
         $this->withType(
             $documentType,
@@ -124,42 +159,47 @@ class Elasticsearch implements Singleton, ConnectionInterface
         );
     }
 
+    public function deleteIndex(string $documentType)
+    {
+        $index = $this->connection->getClient()->getIndex('typo3content');
+
+        if (! $index->exists()) {
+            $this->logger->notice('Index did not exist, therefore was not deleted.', [$documentType, 'typo3content']);
+            return;
+        }
+
+        $index->delete();
+    }
+
     /**
      * Execute given callback with Elastica Type based on provided documentType
-     *
-     * @param string $documentType
-     * @param callable $callback
      */
-    protected function withType($documentType, callable $callback)
+    protected function withType(string $documentType, callable $callback)
     {
         $type = $this->getType($documentType);
+        // TODO: Check whether it's to heavy to send it so often e.g. for every single document.
+        // Perhaps add command controller to submit mapping?!
+        // Also it's not possible to change mapping without deleting index first.
+        // Mattes told about a solution.
+        // So command looks like the best way so far, except we manage mattes solution.
+        // Still then this should be done once. So perhaps singleton which tracks state and does only once?
+        $this->mappingFactory->getMapping($type)->send();
         $callback($type);
         $type->getIndex()->refresh();
     }
 
-    /**
+    public function search(SearchRequestInterface $searchRequest) : SearchResultInterface
-     * @param SearchRequestInterface $searchRequest
-     *
-     * @return \Elastica\ResultSet
-     */
-    public function search(SearchRequestInterface $searchRequest)
     {
         $this->logger->debug('Search for', [$searchRequest->getSearchTerm()]);
 
         $search = new \Elastica\Search($this->connection->getClient());
         $search->addIndex('typo3content');
+        $search->setQuery($this->queryFactory->create($searchRequest));
 
-        // TODO: Return wrapped result to implement our interface.
+        return $this->objectManager->get(SearchResult::class, $searchRequest, $search->search());
-        // Also update php doc to reflect the change.
-        return $search->search('"' . $searchRequest->getSearchTerm() . '"');
     }
 
-    /**
+    protected function getType(string $documentType) : \Elastica\Type
-     * @param string $documentType
-     *
-     * @return \Elastica\Type
-     */
-    protected function getType($documentType)
     {
         return $this->typeFactory->getType(
             $this->indexFactory->getIndex(

Classes/Connection/Elasticsearch/Connection.php

@@ -44,7 +44,7 @@ class Connection implements Singleton
 
     /**
      * @param ConfigurationContainerInterface $configuration
-     * @param \Elastica\Client $elasticaClient
+     * @param \Elastica\Client|null $elasticaClient
      */
     public function __construct(
         ConfigurationContainerInterface $configuration,
@@ -52,9 +52,8 @@ class Connection implements Singleton
     ) {
         $this->configuration = $configuration;
 
-        $this->elasticaClient = $elasticaClient;
+        if ($elasticaClient === null) {
-        if ($this->elasticaClient === null) {
+            $elasticaClient = new \Elastica\Client([
-            $this->elasticaClient = new \Elastica\Client([
                 'host' => $this->configuration->get('connections.elasticsearch.host'),
                 'port' => $this->configuration->get('connections.elasticsearch.port'),
                 // TODO: Make configurable
@@ -63,14 +62,13 @@ class Connection implements Singleton
             // TODO: Make configurable.
             // new \Elastica\Log($this->elasticaClient);
         }
+        $this->elasticaClient = $elasticaClient;
     }
 
     /**
      * Get the concrete client for internal usage!
-     *
-     * @return \Elastica\Client
      */
-    public function getClient()
+    public function getClient() : \Elastica\Client
     {
         return $this->elasticaClient;
     }

Classes/Connection/Elasticsearch/DocumentFactory.php

@@ -44,13 +44,8 @@ class DocumentFactory implements Singleton
 
     /**
      * Creates document from document.
-     *
-     * @param string $documentType
-     * @param array $document
-     *
-     * @return \Elastica\Document
      */
-    public function getDocument($documentType, array $document)
+    public function getDocument(string $documentType, array $document) : \Elastica\Document
     {
         // TODO: Use DocumentType for further configuration.
 
@@ -61,19 +56,17 @@ class DocumentFactory implements Singleton
         $identifier = $document['search_identifier'];
         unset($document['search_identifier']);
 
-        $this->logger->debug('Convert document to document', [$identifier, $document]);
+        $this->logger->debug(
+            sprintf('Convert %s %u to document.', $documentType, $identifier),
+            [$identifier, $document]
+        );
         return new \Elastica\Document($identifier, $document);
     }
 
     /**
      * Creates documents based on documents.
-     *
-     * @param string $documentType
-     * @param array $documents
-     *
-     * @return array
      */
-    public function getDocuments($documentType, array $documents)
+    public function getDocuments(string $documentType, array $documents) : array
     {
         foreach ($documents as &$document) {
             $document = $this->getDocument($documentType, $document);

Classes/Connection/Elasticsearch/Facet.php

@@ -0,0 +1,95 @@
+<?php
+namespace Codappix\SearchCore\Connection\Elasticsearch;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
+use Codappix\SearchCore\Connection\FacetInterface;
+use Codappix\SearchCore\Connection\FacetOptionInterface;
+
+class Facet implements FacetInterface
+{
+    /**
+     * @var string
+     */
+    protected $name = '';
+
+    /**
+     * @var string
+     */
+    protected $field = '';
+
+    /**
+     * @var array
+     */
+    protected $buckets = [];
+
+    /**
+     * @var array<FacetOption>
+     */
+    protected $options = [];
+
+    public function __construct(string $name, array $aggregation, ConfigurationContainerInterface $configuration)
+    {
+        $this->name = $name;
+        $this->buckets = $aggregation['buckets'];
+
+        $config = $configuration->getIfExists('searching.facets.' . $this->name) ?: [];
+        foreach ($config as $configEntry) {
+            if (isset($configEntry['field'])) {
+                $this->field = $configEntry['field'];
+                break;
+            }
+        }
+    }
+
+    public function getName() : string
+    {
+        return $this->name;
+    }
+
+    public function getField() : string
+    {
+        return $this->field;
+    }
+
+    /**
+     * Returns all possible options for this facet.
+     *
+     * @return array<FacetOptionInterface>
+     */
+    public function getOptions() : array
+    {
+        $this->initOptions();
+
+        return $this->options;
+    }
+
+    protected function initOptions()
+    {
+        if ($this->options !== []) {
+            return;
+        }
+
+        foreach ($this->buckets as $bucket) {
+            $this->options[$bucket['key']] = new FacetOption($bucket);
+        }
+    }
+}

Classes/Connection/Elasticsearch/FacetOption.php

@@ -0,0 +1,66 @@
+<?php
+namespace Codappix\SearchCore\Connection\Elasticsearch;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\SearchCore\Connection\FacetOptionInterface;
+
+class FacetOption implements FacetOptionInterface
+{
+    /**
+     * @var string
+     */
+    protected $name = '';
+
+    /**
+     * @var string
+     */
+    protected $displayName = '';
+
+    /**
+     * @var int
+     */
+    protected $count = 0;
+
+    /**
+     * @param array $bucket
+     */
+    public function __construct(array $bucket)
+    {
+        $this->name = $bucket['key'];
+        $this->displayName = isset($bucket['key_as_string']) ? $bucket['key_as_string'] : $this->getName();
+        $this->count = $bucket['doc_count'];
+    }
+
+    public function getName() : string
+    {
+        return $this->name;
+    }
+
+    public function getDisplayName() : string
+    {
+        return $this->displayName;
+    }
+
+    public function getCount() : int
+    {
+        return $this->count;
+    }
+}

Classes/Connection/Elasticsearch/IndexFactory.php

@@ -20,9 +20,10 @@ namespace Codappix\SearchCore\Connection\Elasticsearch;
  * 02110-1301, USA.
  */
 
-use Elastica\Exception\ResponseException;
+use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
+use Codappix\SearchCore\Configuration\InvalidArgumentException;
 use TYPO3\CMS\Core\SingletonInterface as Singleton;
-use TYPO3\CMS\Extbase\Configuration\ConfigurationManagerInterface;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
 
 /**
  * Factory to get indexes.
@@ -32,28 +33,81 @@ use TYPO3\CMS\Extbase\Configuration\ConfigurationManagerInterface;
 class IndexFactory implements Singleton
 {
     /**
-     * Get an index bases on TYPO3 table name.
+     * @var ConfigurationContainerInterface
-     *
-     * @param Connection $connection
-     * @param string $documentType
-     *
-     * @return \Elastica\Index
      */
-    public function getIndex(Connection $connection, $documentType)
+    protected $configuration;
+
+    /**
+     * @var \TYPO3\CMS\Core\Log\Logger
+     */
+    protected $logger;
+
+    /**
+     * Inject log manager to get concrete logger from it.
+     *
+     * @param \TYPO3\CMS\Core\Log\LogManager $logManager
+     */
+    public function injectLogger(\TYPO3\CMS\Core\Log\LogManager $logManager)
+    {
+        $this->logger = $logManager->getLogger(__CLASS__);
+    }
+
+    /**
+     * @param ConfigurationContainerInterface $configuration
+     */
+    public function __construct(ConfigurationContainerInterface $configuration)
+    {
+        $this->configuration = $configuration;
+    }
+
+    /**
+     * Get an index bases on TYPO3 table name.
+     */
+    public function getIndex(Connection $connection, string $documentType) : \Elastica\Index
     {
-        // TODO: Fetch index name from configuration, based on $documentType.
         $index = $connection->getClient()->getIndex('typo3content');
 
-        try {
+        if ($index->exists() === false) {
-            // TODO: Provide configuration?!
+            $config = $this->getConfigurationFor($documentType);
-            // http://elastica.io/getting-started/storing-and-indexing-documents.html#section-analysis
+            $this->logger->debug(sprintf('Create index %s.', $documentType), [$documentType, $config]);
-            $index->create();
+            $index->create($config);
-        } catch (ResponseException $exception) {
+            $this->logger->debug(sprintf('Created index %s.', $documentType), [$documentType]);
-            if (stripos($exception->getMessage(), 'already exists') === false) {
-                throw $exception;
-            }
         }
 
         return $index;
     }
+
+    protected function getConfigurationFor(string $documentType) : array
+    {
+        try {
+            $configuration = $this->configuration->get('indexing.' . $documentType . '.index');
+
+            foreach (['analyzer', 'filter'] as $optionsToExpand) {
+                if (isset($configuration['analysis'][$optionsToExpand])) {
+                    foreach ($configuration['analysis'][$optionsToExpand] as $key => $options) {
+                        $configuration['analysis'][$optionsToExpand][$key] = $this->prepareAnalyzerConfiguration(
+                            $options
+                        );
+                    }
+                }
+            }
+
+            return $configuration;
+        } catch (InvalidArgumentException $e) {
+            return [];
+        }
+    }
+
+    protected function prepareAnalyzerConfiguration(array $analyzer) : array
+    {
+        $fieldsToExplode = ['char_filter', 'filter', 'word_list'];
+
+        foreach ($fieldsToExplode as $fieldToExplode) {
+            if (isset($analyzer[$fieldToExplode])) {
+                $analyzer[$fieldToExplode] = GeneralUtility::trimExplode(',', $analyzer[$fieldToExplode], true);
+            }
+        }
+
+        return $analyzer;
+    }
 }

Classes/Connection/Elasticsearch/MappingFactory.php

@@ -0,0 +1,71 @@
+<?php
+namespace Codappix\SearchCore\Connection\Elasticsearch;
+
+/*
+ * Copyright (C) 2016  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
+use Codappix\SearchCore\Configuration\InvalidArgumentException;
+use TYPO3\CMS\Core\SingletonInterface as Singleton;
+
+/**
+ * Factory to get mappings.
+ */
+class MappingFactory implements Singleton
+{
+    /**
+     * @var ConfigurationContainerInterface
+     */
+    protected $configuration;
+
+    /**
+     * @param ConfigurationContainerInterface $configuration
+     */
+    public function __construct(ConfigurationContainerInterface $configuration)
+    {
+        $this->configuration = $configuration;
+    }
+
+    /**
+     * Get an mapping based on type.
+     */
+    public function getMapping(\Elastica\Type $type) : \Elastica\Type\Mapping
+    {
+        $mapping = new \Elastica\Type\Mapping();
+        $mapping->setType($type);
+
+        $configuration = $this->getConfiguration($type->getName());
+        if (isset($configuration['_all'])) {
+            $mapping->setAllField($configuration['_all']);
+            unset($configuration['_all']);
+        }
+        $mapping->setProperties($configuration);
+
+        return $mapping;
+    }
+
+    protected function getConfiguration(string $identifier) : array
+    {
+        try {
+            return $this->configuration->get('indexing.' . $identifier . '.mapping');
+        } catch (InvalidArgumentException $e) {
+            return [];
+        }
+    }
+}

Classes/Connection/Elasticsearch/SearchResult.php

@@ -20,12 +20,145 @@ namespace Codappix\SearchCore\Connection\Elasticsearch;
  * 02110-1301, USA.
  */
 
+use Codappix\SearchCore\Connection\FacetInterface;
+use Codappix\SearchCore\Connection\ResultItemInterface;
+use Codappix\SearchCore\Connection\SearchRequestInterface;
 use Codappix\SearchCore\Connection\SearchResultInterface;
+use Codappix\SearchCore\Domain\Model\QueryResultInterfaceStub;
+use Codappix\SearchCore\Domain\Model\ResultItem;
+use TYPO3\CMS\Extbase\Object\ObjectManagerInterface;
 
-/**
+class SearchResult implements SearchResultInterface
- *
- */
-class SearchResult extends \Elastica\SearchResult implements SearchResultInterface
 {
+    use QueryResultInterfaceStub;
 
+    /**
+     * @var SearchRequestInterface
+     */
+    protected $searchRequest;
+
+    /**
+     * @var \Elastica\ResultSet
+     */
+    protected $result;
+
+    /**
+     * @var array<FacetInterface>
+     */
+    protected $facets = [];
+
+    /**
+     * @var array<ResultItemInterface>
+     */
+    protected $results = [];
+
+    /**
+     * For Iterator interface.
+     *
+     * @var int
+     */
+    protected $position = 0;
+
+    /**
+     * @var ObjectManagerInterface
+     */
+    protected $objectManager;
+
+    public function __construct(
+        SearchRequestInterface $searchRequest,
+        \Elastica\ResultSet $result,
+        ObjectManagerInterface $objectManager
+    ) {
+        $this->searchRequest = $searchRequest;
+        $this->result = $result;
+        $this->objectManager = $objectManager;
+    }
+
+    /**
+     * @return array<ResultItemInterface>
+     */
+    public function getResults() : array
+    {
+        $this->initResults();
+
+        return $this->results;
+    }
+
+    /**
+     * Return all facets, if any.
+     *
+     * @return array<FacetInterface>
+     */
+    public function getFacets() : array
+    {
+        $this->initFacets();
+
+        return $this->facets;
+    }
+
+    public function getCurrentCount() : int
+    {
+        return $this->result->count();
+    }
+
+    protected function initResults()
+    {
+        if ($this->results !== []) {
+            return;
+        }
+
+        foreach ($this->result->getResults() as $result) {
+            $this->results[] = new ResultItem($result->getData());
+        }
+    }
+
+    protected function initFacets()
+    {
+        if ($this->facets !== [] || !$this->result->hasAggregations()) {
+            return;
+        }
+
+        foreach ($this->result->getAggregations() as $aggregationName => $aggregation) {
+            $this->facets[$aggregationName] = $this->objectManager->get(Facet::class, $aggregationName, $aggregation);
+        }
+    }
+
+    // Countable - Interface
+    public function count()
+    {
+        return $this->result->getTotalHits();
+    }
+
+    // Iterator - Interface
+    public function current()
+    {
+        return $this->getResults()[$this->position];
+    }
+
+    public function next()
+    {
+        ++$this->position;
+
+        return $this->current();
+    }
+
+    public function key()
+    {
+        return $this->position;
+    }
+
+    public function valid()
+    {
+        return isset($this->getResults()[$this->position]);
+    }
+
+    public function rewind()
+    {
+        $this->position = 0;
+    }
+
+    public function getQuery()
+    {
+        return $this->searchRequest;
+    }
 }

Classes/Connection/Elasticsearch/TypeFactory.php

@@ -21,7 +21,6 @@ namespace Codappix\SearchCore\Connection\Elasticsearch;
  */
 
 use TYPO3\CMS\Core\SingletonInterface as Singleton;
-use TYPO3\CMS\Extbase\Configuration\ConfigurationManagerInterface;
 
 /**
  * Factory to get indexes.
@@ -32,13 +31,8 @@ class TypeFactory implements Singleton
 {
     /**
      * Get an index bases on TYPO3 table name.
-     *
-     * @param \Elastica\Index $index
-     * @param string $documentType
-     *
-     * @return \Elastica\Type
      */
-    public function getType(\Elastica\Index $index, $documentType)
+    public function getType(\Elastica\Index $index, string $documentType) : \Elastica\Type
     {
         return $index->getType($documentType);
     }

Classes/Connection/FacetInterface.php

@@ -0,0 +1,36 @@
+<?php
+namespace Codappix\SearchCore\Connection;
+
+/*
+ * Copyright (C) 2016  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * A single facet, e.g. keyword based.
+ */
+interface FacetInterface
+{
+    public function getName() : string;
+
+    /**
+     * Returns all possible options for this facet.
+     *
+     * @return array<FacetOptionInterface>
+     */
+    public function getOptions() : array;
+}

Classes/Connection/FacetOptionInterface.php

@@ -0,0 +1,44 @@
+<?php
+namespace Codappix\SearchCore\Connection;
+
+/*
+ * Copyright (C) 2016  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * A single possible option of a facet.
+ */
+interface FacetOptionInterface
+{
+    /**
+     * Returns the name of this option. Equivalent
+     * to value used for filtering.
+     */
+    public function getName() : string;
+
+    /**
+     * If a pre-rendered name is provided, this will be returned.
+     * Otherwise it's the same as getName().
+     */
+    public function getDisplayName() : string;
+
+    /**
+     * Returns the number of found results for this option.
+     */
+    public function getCount() : int;
+}

Classes/Connection/FacetRequestInterface.php

@@ -0,0 +1,38 @@
+<?php
+namespace Codappix\SearchCore\Connection;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * Used to request facets / aggregates from connection.
+ */
+interface FacetRequestInterface
+{
+    /**
+     * The identifier of the facet, used as key in arrays and to get the facet
+     * from search request, etc.
+     */
+    public function getIdentifier() : string;
+
+    /**
+     * The config to use for facet building.
+     */
+    public function getConfig() : array;
+}

Classes/Connection/ResultItemInterface.php

@@ -0,0 +1,36 @@
+<?php
+namespace Codappix\SearchCore\Connection;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * Use ArrayAccess to enable retrieval of information in fluid.
+ */
+interface ResultItemInterface extends \ArrayAccess
+{
+    /**
+     * Returns every information as array.
+     *
+     * Provide key/column/field => data.
+     *
+     * Used e.g. for dataprocessing.
+     */
+    public function getPlainData() : array;
+}

Classes/Connection/SearchRequestInterface.php

@@ -20,15 +20,47 @@ namespace Codappix\SearchCore\Connection;
  * 02110-1301, USA.
  */
 
-/**
+use Codappix\SearchCore\Connection\ConnectionInterface;
- *
+use Codappix\SearchCore\Connection\FacetRequestInterface;
- */
+use Codappix\SearchCore\Domain\Search\SearchService;
-interface SearchRequestInterface
+use TYPO3\CMS\Extbase\Persistence\QueryInterface;
+
+interface SearchRequestInterface extends QueryInterface
 {
     /**
      * Returns the actual string the user searched for.
-     *
-     * @return string
      */
-    public function getSearchTerm();
+    public function getSearchTerm() : string;
+
+    public function hasFilter() : bool;
+
+    public function getFilter() : array;
+
+    public function setFilter(array $filter);
+
+    /**
+     * @return void
+     */
+    public function addFacet(FacetRequestInterface $facet);
+
+    /**
+     * @return array<FacetRequestInterface>
+     */
+    public function getFacets() : array;
+
+    /**
+     * Workaround for paginate widget support which will
+     * use the request to build another search.
+     *
+     * @return void
+     */
+    public function setConnection(ConnectionInterface $connection);
+
+    /**
+     * Workaround for paginate widget support which will
+     * use the request to build another search.
+     *
+     * @return void
+     */
+    public function setSearchService(SearchService $searchService);
 }

Classes/Connection/SearchResultInterface.php

@@ -20,10 +20,27 @@ namespace Codappix\SearchCore\Connection;
  * 02110-1301, USA.
  */
 
-/**
+use TYPO3\CMS\Extbase\Persistence\QueryResultInterface;
- *
- */
-interface SearchResultInterface extends \Iterator, \Countable, \ArrayAccess
-{
 
+/**
+ * A search result.
+ */
+interface SearchResultInterface extends \Iterator, \Countable, QueryResultInterface
+{
+    /**
+     * @return array<ResultItemInterface>
+     */
+    public function getResults() : array;
+
+    /**
+     * Return all facets, if any.
+     *
+     * @return array<FacetInterface>
+     */
+    public function getFacets() : array;
+
+    /**
+     * Returns the number of results in current result
+     */
+    public function getCurrentCount() : int;
 }

Classes/Controller/SearchController.php

@@ -44,6 +44,26 @@ class SearchController extends ActionController
         parent::__construct();
     }
 
+    public function initializeSearchAction()
+    {
+        if (isset($this->settings['searching']['mode']) && $this->settings['searching']['mode'] === 'filter'
+            && $this->request->hasArgument('searchRequest') === false
+        ) {
+            $this->request->setArguments(array_merge(
+                $this->request->getArguments(),
+                [
+                    'searchRequest' => $this->objectManager->get(SearchRequest::class),
+                ]
+            ));
+        }
+
+        if ($this->arguments->hasArgument('searchRequest')) {
+            $this->arguments->getArgument('searchRequest')->getPropertyMappingConfiguration()
+                ->allowAllProperties()
+                ;
+        }
+    }
+
     /**
      * Process a search and deliver original request and result to view.
      *

Classes/DataProcessing/ContentObjectDataProcessorAdapterProcessor.php

@@ -0,0 +1,59 @@
+<?php
+namespace Codappix\SearchCore\DataProcessing;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\SearchCore\Compatibility\TypoScriptServiceInterface;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Frontend\ContentObject\ContentObjectRenderer;
+
+/**
+ * Executes an existing TYPO3 DataProcessor on the given data.
+ */
+class ContentObjectDataProcessorAdapterProcessor implements ProcessorInterface
+{
+    /**
+     * @var TypoScriptServiceInterface
+     */
+    protected $typoScriptService;
+
+    public function __construct(TypoScriptServiceInterface $typoScriptService)
+    {
+        $this->typoScriptService = $typoScriptService;
+    }
+
+    public function processData(array $data, array $configuration) : array
+    {
+        $dataProcessor = GeneralUtility::makeInstance($configuration['_dataProcessor']);
+        $contentObjectRenderer = GeneralUtility::makeInstance(ContentObjectRenderer::class);
+
+        $contentObjectRenderer->data = $data;
+        if (isset($configuration['_table'])) {
+            $contentObjectRenderer->start($data, $configuration['_table']);
+        }
+
+        return $dataProcessor->process(
+            $contentObjectRenderer,
+            [],
+            $this->typoScriptService->convertPlainArrayToTypoScriptArray($configuration),
+            $data
+        );
+    }
+}

Classes/DataProcessing/CopyToProcessor.php

@@ -0,0 +1,50 @@
+<?php
+namespace Codappix\SearchCore\DataProcessing;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * Copies values from one field to another one.
+ */
+class CopyToProcessor implements ProcessorInterface
+{
+    public function processData(array $record, array $configuration) : array
+    {
+        $all = [];
+
+        $this->addArray($all, $record);
+        $all = array_filter($all);
+        $record[$configuration['to']] = implode(PHP_EOL, $all);
+
+        return $record;
+    }
+
+    protected function addArray(array &$target, array $from)
+    {
+        foreach ($from as $value) {
+            if (is_array($value)) {
+                $this->addArray($target, $value);
+                continue;
+            }
+
+            $target[] = (string) $value;
+        }
+    }
+}

Classes/DataProcessing/GeoPointProcessor.php

@@ -0,0 +1,59 @@
+<?php
+namespace Codappix\SearchCore\DataProcessing;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * Adds a new fields, ready to use as GeoPoint field for Elasticsearch.
+ */
+class GeoPointProcessor implements ProcessorInterface
+{
+    public function processData(array $record, array $configuration) : array
+    {
+        if (! $this->isApplyable($record, $configuration)) {
+            return $record;
+        }
+
+        $record[$configuration['to']] = [
+            'lat' => (float) $record[$configuration['lat']],
+            'lon' => (float) $record[$configuration['lon']],
+        ];
+
+        return $record;
+    }
+
+    protected function isApplyable(array $record, array $configuration) : bool
+    {
+        if (!isset($record[$configuration['lat']])
+            || !is_numeric($record[$configuration['lat']])
+            || trim($record[$configuration['lat']]) === ''
+        ) {
+            return false;
+        }
+        if (!isset($record[$configuration['lon']])
+            || !is_numeric($record[$configuration['lon']])
+            || trim($record[$configuration['lon']]) === ''
+        ) {
+            return false;
+        }
+
+        return true;
+    }
+}

Classes/DataProcessing/ProcessorInterface.php

@@ -0,0 +1,33 @@
+<?php
+namespace Codappix\SearchCore\DataProcessing;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * All DataProcessing Processors should implement this interface.
+ */
+interface ProcessorInterface
+{
+    /**
+     * Processes the given data.
+     * Also retrieves the configuration for this processor instance.
+     */
+    public function processData(array $record, array $configuration) : array;
+}

Classes/DataProcessing/RemoveProcessor.php

@@ -0,0 +1,44 @@
+<?php
+namespace Codappix\SearchCore\DataProcessing;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+/**
+ * Removes fields from record.
+ */
+class RemoveProcessor implements ProcessorInterface
+{
+    public function processData(array $record, array $configuration) : array
+    {
+        if (!isset($configuration['fields'])) {
+            return $record;
+        }
+
+        foreach (GeneralUtility::trimExplode(',', $configuration['fields'], true) as $field) {
+            if (array_key_exists($field, $record)) {
+                unset($record[$field]);
+            }
+        }
+
+        return $record;
+    }
+}

Classes/DataProcessing/Service.php

@@ -0,0 +1,56 @@
+<?php
+namespace Codappix\SearchCore\DataProcessing;
+
+/*
+ * Copyright (C) 2018  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use TYPO3\CMS\Extbase\Object\ObjectManagerInterface;
+
+/**
+ * Eases work with data processing.
+ */
+class Service
+{
+    /**
+     * @var ObjectManagerInterface
+     */
+    protected $objectManager;
+
+    public function __construct(ObjectManagerInterface $objectManager)
+    {
+        $this->objectManager = $objectManager;
+    }
+
+    /**
+     * Executes the dataprocessor depending on configuration and returns the result.
+     *
+     * @param array|string $configuration Either the full configuration or only the class name.
+     */
+    public function executeDataProcessor($configuration, array $data) : array
+    {
+        if (is_string($configuration)) {
+            $configuration = [
+                '_typoScriptNodeValue' => $configuration,
+            ];
+        }
+
+        return $this->objectManager->get($configuration['_typoScriptNodeValue'])
+            ->processData($data, $configuration);
+    }
+}

Classes/Database/Doctrine/Join.php

@@ -0,0 +1,50 @@
+<?php
+namespace Codappix\SearchCore\Database\Doctrine;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+class Join
+{
+    /**
+     * @var string
+     */
+    protected $table = '';
+
+    /**
+     * @var string
+     */
+    protected $condition = '';
+
+    public function __construct(string $table, string $condition)
+    {
+        $this->table = $table;
+        $this->condition = $condition;
+    }
+
+    public function getTable() : string
+    {
+        return $this->table;
+    }
+
+    public function getCondition() : string
+    {
+        return $this->condition;
+    }
+}

Classes/Database/Doctrine/Where.php

@@ -0,0 +1,50 @@
+<?php
+namespace Codappix\SearchCore\Database\Doctrine;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+class Where
+{
+    /**
+     * @var string
+     */
+    protected $statement = '';
+
+    /**
+     * @var array
+     */
+    protected $parameters = [];
+
+    public function __construct(string $statement, array $parameters)
+    {
+        $this->statement = $statement;
+        $this->parameters = $parameters;
+    }
+
+    public function getStatement() : string
+    {
+        return $this->statement;
+    }
+
+    public function getParameters() : array
+    {
+        return $this->parameters;
+    }
+}

Classes/Domain/Index/AbstractIndexer.php

@@ -0,0 +1,184 @@
+<?php
+namespace Codappix\SearchCore\Domain\Index;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
+use Codappix\SearchCore\Configuration\InvalidArgumentException;
+use Codappix\SearchCore\Connection\ConnectionInterface;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+abstract class AbstractIndexer implements IndexerInterface
+{
+    /**
+     * @var ConnectionInterface
+     */
+    protected $connection;
+
+    /**
+     * @var ConfigurationContainerInterface
+     */
+    protected $configuration;
+
+    /**
+     * @var string
+     */
+    protected $identifier = '';
+
+    /**
+     * @var \Codappix\SearchCore\DataProcessing\Service
+     * @inject
+     */
+    protected $dataProcessorService;
+
+    /**
+     * @var \TYPO3\CMS\Core\Log\Logger
+     */
+    protected $logger;
+
+    /**
+     * Inject log manager to get concrete logger from it.
+     *
+     * @param \TYPO3\CMS\Core\Log\LogManager $logManager
+     */
+    public function injectLogger(\TYPO3\CMS\Core\Log\LogManager $logManager)
+    {
+        $this->logger = $logManager->getLogger(__CLASS__);
+    }
+
+    public function setIdentifier(string $identifier)
+    {
+        $this->identifier = $identifier;
+    }
+
+    public function __construct(ConnectionInterface $connection, ConfigurationContainerInterface $configuration)
+    {
+        $this->connection = $connection;
+        $this->configuration = $configuration;
+    }
+
+    public function indexAllDocuments()
+    {
+        $this->logger->info('Start indexing');
+        foreach ($this->getRecordGenerator() as $records) {
+            if ($records === null) {
+                break;
+            }
+
+            foreach ($records as &$record) {
+                $this->prepareRecord($record);
+            }
+
+            $this->logger->debug('Index records.', [$records]);
+            $this->connection->addDocuments($this->getDocumentName(), $records);
+        }
+        $this->logger->info('Finish indexing');
+    }
+
+    public function indexDocument(string $identifier)
+    {
+        $this->logger->info('Start indexing single record.', [$identifier]);
+        try {
+            $record = $this->getRecord((int) $identifier);
+            $this->prepareRecord($record);
+
+            $this->connection->addDocument($this->getDocumentName(), $record);
+        } catch (NoRecordFoundException $e) {
+            $this->logger->info('Could not index document. Try to delete it therefore.', [$e->getMessage()]);
+            $this->connection->deleteDocument($this->getDocumentName(), $identifier);
+        }
+        $this->logger->info('Finish indexing');
+    }
+
+    public function delete()
+    {
+        $this->logger->info('Start deletion of index.');
+        $this->connection->deleteIndex($this->getDocumentName());
+        $this->logger->info('Finish deletion.');
+    }
+
+    protected function getRecordGenerator() : \Generator
+    {
+        $offset = 0;
+        $limit = $this->getLimit();
+
+        while (($records = $this->getRecords($offset, $limit)) !== []) {
+            yield $records;
+            $offset += $limit;
+        }
+    }
+
+    protected function prepareRecord(array &$record)
+    {
+        try {
+            foreach ($this->configuration->get('indexing.' . $this->identifier . '.dataProcessing') as $configuration) {
+                $record = $this->dataProcessorService->executeDataProcessor($configuration, $record);
+            }
+        } catch (InvalidArgumentException $e) {
+            // Nothing to do.
+        }
+
+        $this->handleAbstract($record);
+    }
+
+    protected function handleAbstract(array &$record)
+    {
+        $record['search_abstract'] = '';
+
+        try {
+            $fieldsToUse = GeneralUtility::trimExplode(
+                ',',
+                $this->configuration->get('indexing.' . $this->identifier . '.abstractFields')
+            );
+            if ($fieldsToUse === []) {
+                return;
+            }
+            foreach ($fieldsToUse as $fieldToUse) {
+                if (isset($record[$fieldToUse]) && trim($record[$fieldToUse])) {
+                    $record['search_abstract'] = trim($record[$fieldToUse]);
+                    break;
+                }
+            }
+        } catch (InvalidArgumentException $e) {
+            return;
+        }
+    }
+
+    /**
+     * Returns the limit to use to fetch records.
+     */
+    protected function getLimit() : int
+    {
+        // TODO: Make configurable.
+        return 50;
+    }
+
+    /**
+     * @return array|null
+     */
+    abstract protected function getRecords(int $offset, int $limit);
+
+    /**
+     * @throws NoRecordFoundException If record could not be found.
+     */
+    abstract protected function getRecord(int $identifier) : array;
+
+    abstract protected function getDocumentName() : string;
+}

Classes/Domain/Index/IndexerFactory.php

@@ -20,6 +20,10 @@ namespace Codappix\SearchCore\Domain\Index;
  * 02110-1301, USA.
  */
 
+use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
+use Codappix\SearchCore\Configuration\InvalidArgumentException;
+use Codappix\SearchCore\Domain\Index\IndexerInterface;
+use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface;
 use TYPO3\CMS\Core\SingletonInterface as Singleton;
 use TYPO3\CMS\Extbase\Object\ObjectManagerInterface;
 
@@ -34,27 +38,67 @@ class IndexerFactory implements Singleton
     protected $objectManager;
 
     /**
-     * @param ObjectManagerInterface $objectManager
+     * @var ConfigurationContainerInterface
      */
-    public function __construct(ObjectManagerInterface $objectManager)
+    protected $configuration;
-    {
+
+    /**
+     * @param ObjectManagerInterface $objectManager
+     * @param ConfigurationContainerInterface $configuration
+     */
+    public function __construct(
+        ObjectManagerInterface $objectManager,
+        ConfigurationContainerInterface $configuration
+    ) {
         $this->objectManager = $objectManager;
+        $this->configuration = $configuration;
     }
 
     /**
-     * @param string $tableName
+     * @throws NoMatchingIndexer
-     *
-     * @return IndexerInterface
      */
-    public function getIndexer($tableName)
+    public function getIndexer(string $identifier) : IndexerInterface
     {
-        // This is the place to use configuration to return different indexer.
+        try {
-        return $this->objectManager->get(
+            return $this->buildIndexer($this->configuration->get('indexing.' . $identifier . '.indexer'), $identifier);
-            TcaIndexer::Class,
+        } catch (NoMatchingIndexerException $e) {
-            $this->objectManager->get(
+            // Nothing to do, we throw exception below
-                TcaIndexer\TcaTableService::class,
+        } catch (InvalidArgumentException $e) {
-                $tableName
+            // Nothing to do, we throw exception below
-            )
+        }
-        );
+
+        throw new NoMatchingIndexerException('Could not find an indexer for ' . $identifier, 1497341442);
+    }
+
+    /**
+     * @throws NoMatchingIndexer
+     */
+    protected function buildIndexer(string $indexerClass, string $identifier) : IndexerInterface
+    {
+        $indexer = null;
+        if (is_subclass_of($indexerClass, TcaIndexer\PagesIndexer::class)
+            || $indexerClass === TcaIndexer\PagesIndexer::class
+        ) {
+            $indexer = $this->objectManager->get(
+                $indexerClass,
+                $this->objectManager->get(TcaTableServiceInterface::class, $identifier),
+                $this->objectManager->get(TcaTableServiceInterface::class, 'tt_content')
+            );
+        } elseif (is_subclass_of($indexerClass, TcaIndexer::class) || $indexerClass === TcaIndexer::class) {
+            $indexer = $this->objectManager->get(
+                $indexerClass,
+                $this->objectManager->get(TcaTableServiceInterface::class, $identifier)
+            );
+        } elseif (class_exists($indexerClass) && in_array(IndexerInterface::class, class_implements($indexerClass))) {
+            $indexer = $this->objectManager->get($indexerClass);
+        }
+
+        if ($indexer === null) {
+            throw new NoMatchingIndexerException('Could not find indexer: ' . $indexerClass, 1497341442);
+        }
+
+        $indexer->setIdentifier($identifier);
+
+        return $indexer;
     }
 }

Classes/Domain/Index/IndexerInterface.php

@@ -33,11 +33,23 @@ interface IndexerInterface
     public function indexAllDocuments();
 
     /**
-     * Fetches a single document from the indexerService and pushes it to the connection.
+     * Fetches a single document and pushes it to the connection.
-     *
-     * @param string $identifier identifier, the indexer needs to identify a single document
      *
      * @return void
      */
-    public function indexDocument($identifier);
+    public function indexDocument(string $identifier);
+
+    /**
+     * Recieves the identifier of the indexer itself.
+     *
+     * @return void
+     */
+    public function setIdentifier(string $identifier);
+
+    /**
+     * Delete the whole index.
+     *
+     * @return void
+     */
+    public function delete();
 }

Classes/Domain/Index/NoMatchingIndexerException.php

@@ -0,0 +1,25 @@
+<?php
+namespace Codappix\SearchCore\Domain\Index;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+class NoMatchingIndexerException extends IndexingException
+{
+}

Classes/Domain/Index/TcaIndexer.php

@@ -20,107 +20,41 @@ namespace Codappix\SearchCore\Domain\Index;
  * 02110-1301, USA.
  */
 
-use TYPO3\CMS\Extbase\Object\ObjectManagerInterface;
+use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
 use Codappix\SearchCore\Connection\ConnectionInterface;
+use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface;
 
 /**
  * Will index the given table using configuration from TCA.
  */
-class TcaIndexer implements IndexerInterface
+class TcaIndexer extends AbstractIndexer
 {
     /**
-     * @var ConnectionInterface
+     * @var TcaTableServiceInterface
-     */
-    protected $connection;
-
-    /**
-     * @var TcaIndexer\TcaTableService
      */
     protected $tcaTableService;
 
     /**
-     * @var \TYPO3\CMS\Core\Log\Logger
+     * @param TcaTableServiceInterface $tcaTableService
-     */
-    protected $logger;
-
-    /**
-     * Inject log manager to get concrete logger from it.
-     *
-     * @param \TYPO3\CMS\Core\Log\LogManager $logManager
-     */
-    public function injectLogger(\TYPO3\CMS\Core\Log\LogManager $logManager)
-    {
-        $this->logger = $logManager->getLogger(__CLASS__);
-    }
-
-    /**
-     * @param TcaIndexer\TcaTableService $tcaTableService
      * @param ConnectionInterface $connection
+     * @param ConfigurationContainerInterface $configuration
      */
     public function __construct(
-        TcaIndexer\TcaTableService $tcaTableService,
+        TcaTableServiceInterface $tcaTableService,
-        ConnectionInterface $connection
+        ConnectionInterface $connection,
+        ConfigurationContainerInterface $configuration
     ) {
+        parent::__construct($connection, $configuration);
         $this->tcaTableService = $tcaTableService;
-        $this->connection = $connection;
-    }
-
-    public function indexAllDocuments()
-    {
-        $this->logger->info('Start indexing');
-        foreach ($this->getRecordGenerator() as $records) {
-            $this->logger->debug('Index records.', [$records]);
-            if ($records === null) {
-                break;
-            }
-
-            $this->connection->addDocuments($this->tcaTableService->getTableName(), $records);
-        }
-        $this->logger->info('Finish indexing');
-    }
-
-    public function indexDocument($identifier)
-    {
-        $this->logger->info('Start indexing single record.', [$identifier]);
-        try {
-            $this->connection->addDocument($this->tcaTableService->getTableName(), $this->getRecord($identifier));
-        } catch (NoRecordFoundException $e) {
-            $this->logger->info('Could not index document.', [$e->getMessage()]);
-        }
-        $this->logger->info('Finish indexing');
     }
 
     /**
-     * @return \Generator
-     */
-    protected function getRecordGenerator()
-    {
-        $offset = 0;
-        // TODO: Make configurable.
-        $limit = 50;
-
-        while (($records = $this->getRecords($offset, $limit)) !== []) {
-            yield $records;
-            $offset += $limit;
-        }
-    }
-
-    /**
-     * @param int $offset
-     * @param int $limit
      * @return array|null
      */
-    protected function getRecords($offset, $limit)
+    protected function getRecords(int $offset, int $limit)
     {
-        $records = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows(
+        $records = $this->tcaTableService->getRecords($offset, $limit);
-            $this->tcaTableService->getFields(),
+        if ($records === []) {
-            $this->tcaTableService->getTableClause(),
-            $this->tcaTableService->getWhereClause(),
-            '',
-            '',
-            (int) $offset . ',' . (int) $limit
-        );
-        if ($records === null) {
             return null;
         }
 
@@ -133,20 +67,13 @@ class TcaIndexer implements IndexerInterface
     }
 
     /**
-     * @param int $identifier
-     * @return array
      * @throws NoRecordFoundException If record could not be found.
      */
-    protected function getRecord($identifier)
+    protected function getRecord(int $identifier) : array
     {
-        $record = $GLOBALS['TYPO3_DB']->exec_SELECTgetSingleRow(
+        $record = $this->tcaTableService->getRecord($identifier);
-            $this->tcaTableService->getFields(),
-            $this->tcaTableService->getTableClause(),
-            $this->tcaTableService->getWhereClause()
-                . ' AND ' . $this->tcaTableService->getTableName() . '.uid = ' . (int) $identifier
-        );
 
-        if ($record === false || $record === null) {
+        if ($record === []) {
             throw new NoRecordFoundException(
                 'Record could not be fetched from database: "' . $identifier . '". Perhaps record is not active.',
                 1484225364
@@ -156,4 +83,9 @@ class TcaIndexer implements IndexerInterface
 
         return $record;
     }
+
+    protected function getDocumentName() : string
+    {
+        return $this->tcaTableService->getTableName();
+    }
 }

Classes/Domain/Index/TcaIndexer/PagesIndexer.php

@@ -0,0 +1,139 @@
+<?php
+namespace Codappix\SearchCore\Domain\Index\TcaIndexer;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
+use Codappix\SearchCore\Connection\ConnectionInterface;
+use Codappix\SearchCore\Domain\Index\TcaIndexer;
+use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService;
+
+/**
+ * Specific indexer for Pages, will basically add content of page.
+ */
+class PagesIndexer extends TcaIndexer
+{
+    /**
+     * @var TcaTableServiceInterface
+     */
+    protected $contentTableService;
+
+    /**
+     * @var \TYPO3\CMS\Core\Resource\FileRepository
+     * @inject
+     */
+    protected $fileRepository;
+
+    /**
+     * @param TcaTableServiceInterface $tcaTableService
+     * @param TcaTableServiceInterface $contentTableService
+     * @param ConnectionInterface $connection
+     * @param ConfigurationContainerInterface $configuration
+     */
+    public function __construct(
+        TcaTableServiceInterface $tcaTableService,
+        TcaTableServiceInterface $contentTableService,
+        ConnectionInterface $connection,
+        ConfigurationContainerInterface $configuration
+    ) {
+        parent::__construct($tcaTableService, $connection, $configuration);
+        $this->contentTableService = $contentTableService;
+    }
+
+    protected function prepareRecord(array &$record)
+    {
+        $possibleTitleFields = ['nav_title', 'tx_tqseo_pagetitle_rel', 'title'];
+        foreach ($possibleTitleFields as $searchTitleField) {
+            if (isset($record[$searchTitleField]) && trim($record[$searchTitleField])) {
+                $record['search_title'] = trim($record[$searchTitleField]);
+                break;
+            }
+        }
+
+        $record['media'] = $this->fetchMediaForPage($record['uid']);
+        $content = $this->fetchContentForPage($record['uid']);
+        if ($content !== []) {
+            $record['content'] = $content['content'];
+            $record['media'] = array_values(array_unique(array_merge($record['media'], $content['images'])));
+        }
+        parent::prepareRecord($record);
+    }
+
+    protected function fetchContentForPage(int $uid) : array
+    {
+        if ($this->contentTableService instanceof TcaTableService) {
+            $contentElements = $this->contentTableService->getQuery()
+                ->execute()->fetchAll();
+        } else {
+            $contentElements = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows(
+                $this->contentTableService->getFields(),
+                $this->contentTableService->getTableClause(),
+                $this->contentTableService->getWhereClause() .
+                sprintf(' AND %s.pid = %u', $this->contentTableService->getTableName(), $uid)
+            );
+        }
+
+        if ($contentElements === null) {
+            $this->logger->debug('No content for page ' . $uid);
+            return [];
+        }
+
+        $this->logger->debug('Fetched content for page ' . $uid);
+        $images = [];
+        $content = [];
+        foreach ($contentElements as $contentElement) {
+            $images = array_merge(
+                $images,
+                $this->getContentElementImages($contentElement['uid'])
+            );
+            $content[] = $contentElement['bodytext'];
+        }
+
+        return [
+            // Remove Tags.
+            // Interpret escaped new lines and special chars.
+            // Trim, e.g. trailing or leading new lines.
+            'content' => trim(stripcslashes(strip_tags(implode(' ', $content)))),
+            'images' => $images,
+        ];
+    }
+
+    protected function getContentElementImages(int $uidOfContentElement) : array
+    {
+        return $this->fetchSysFileReferenceUids($uidOfContentElement, 'tt_content', 'image');
+    }
+
+    protected function fetchMediaForPage(int $uid) : array
+    {
+        return $this->fetchSysFileReferenceUids($uid, 'pages', 'media');
+    }
+
+    protected function fetchSysFileReferenceUids(int $uid, string $tablename, string $fieldname) : array
+    {
+        $imageRelationUids = [];
+        $imageRelations = $this->fileRepository->findByRelation($tablename, $fieldname, $uid);
+
+        foreach ($imageRelations as $relation) {
+            $imageRelationUids[] = $relation->getUid();
+        }
+
+        return $imageRelationUids;
+    }
+}

Classes/Domain/Index/TcaIndexer/RelationResolver.php

@@ -20,13 +20,10 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer;
  * 02110-1301, USA.
  */
 
+use Codappix\SearchCore\Utility\FrontendUtility;
 use TYPO3\CMS\Backend\Utility\BackendUtility;
 use TYPO3\CMS\Core\SingletonInterface as Singleton;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
-use TYPO3\CMS\Extbase\Utility\LocalizationUtility;
-
-use TYPO3\CMS\Backend\Form\FormDataCompiler;
-use TYPO3\CMS\Backend\Form\FormDataGroup\TcaDatabaseRecord;
 
 /**
  * Resolves relations from TCA using TCA.
@@ -36,130 +33,76 @@ use TYPO3\CMS\Backend\Form\FormDataGroup\TcaDatabaseRecord;
  */
 class RelationResolver implements Singleton
 {
-    /**
+    public function resolveRelationsForRecord(TcaTableServiceInterface $service, array &$record)
-     * Resolve relations for the given record.
-     *
-     * @param TcaTableService $service
-     * @param array $record
-     */
-    public function resolveRelationsForRecord(TcaTableService $service, array &$record)
     {
-        $formData = GeneralUtility::makeInstance(
-            FormDataCompiler::class,
-            GeneralUtility::makeInstance(TcaDatabaseRecord::class)
-        )->compile([
-            'tableName' => $service->getTableName(),
-            'vanillaUid' => (int)$record['uid'],
-            'command' => 'edit',
-        ]);
-        $record = $formData['databaseRow'];
-
         foreach (array_keys($record) as $column) {
+            // TODO: Define / configure fields to exclude?!
+            if ($column === 'pid') {
+                continue;
+            }
+
+            $record[$column] = GeneralUtility::makeInstance($this->getUtilityForMode())
+                ::getProcessedValueExtra(
+                    $service->getTableName(),
+                    $column,
+                    $record[$column],
+                    0,
+                    $record['uid']
+                );
+
             try {
                 $config = $service->getColumnConfig($column);
+
+                if ($this->isRelation($config)) {
+                    $record[$column] = $this->resolveValue($record[$column], $config);
+                }
             } catch (InvalidArgumentException $e) {
                 // Column is not configured.
                 continue;
             }
-
-            if (! $this->isRelation($config) || !is_array($formData['processedTca']['columns'][$column])) {
-                continue;
-            }
-
-            $record[$column] = $this->resolveValue($record[$column], $formData['processedTca']['columns'][$column]);
         }
     }
 
-    /**
-     * Resolve the given value from TYPO3 API response.
-     *
-     * @param string $value The value from FormEngine to resolve.
-     * @param array $tcaColumn The tca config of the relation.
-     *
-     * @return array<String>|string
-     */
     protected function resolveValue($value, array $tcaColumn)
     {
-        if ($value === '' || $value === '0') {
+        if ($value === '' || $value === 'N/A') {
-            return '';
+            return [];
         }
-        if ($tcaColumn['config']['type'] === 'select') {
+
-            return $this->resolveSelectValue($value, $tcaColumn);
+        if ($tcaColumn['type'] === 'select' && strpos($value, ';') !== false) {
-        }
-        if ($tcaColumn['config']['type'] === 'group' && strpos($value, '|') !== false) {
             return $this->resolveForeignDbValue($value);
         }
-        if ($tcaColumn['config']['type'] === 'inline') {
+        if (in_array($tcaColumn['type'], ['inline', 'group', 'select'])) {
-            return $this->resolveInlineValue($tcaColumn);
+            return $this->resolveInlineValue($value);
         }
 
-        return '';
+        return [];
     }
 
-    /**
+    protected function isRelation(array &$config) : bool
-     * @param array Column config.
-     * @return bool
-     */
-    protected function isRelation(array &$config)
     {
         return isset($config['foreign_table'])
-            || (isset($config['items']) && is_array($config['items']))
+            || (isset($config['renderType']) && $config['renderType'] !== 'selectSingle')
             || (isset($config['internal_type']) && strtolower($config['internal_type']) === 'db')
             ;
     }
 
-    /**
+    protected function resolveForeignDbValue(string $value) : array
-     * Resolves internal representation of select to array of labels.
-     *
-     * @param array $value
-     * @param array $tcaColumn
-     * @return array
-     */
-    protected function resolveSelectValue(array $values, array $tcaColumn)
     {
-        $resolvedValues = [];
+        return array_map('trim', explode(';', $value));
-
-        foreach ($tcaColumn['config']['items'] as $item) {
-            if (in_array($item[1], $values)) {
-                $resolvedValues[] = $item[0];
-            }
-        }
-
-        if ($tcaColumn['config']['renderType'] === 'selectSingle' || $tcaColumn['config']['maxitems'] === 1) {
-            return current($resolvedValues);
-        }
-
-        return $resolvedValues;
     }
 
-    /**
+    protected function resolveInlineValue(string $value) : array
-     * @param string $value
-     *
-     * @return array
-     */
-    protected function resolveForeignDbValue($value)
     {
-        $titles = [];
+        return array_map('trim', explode(',', $value));
-
-        foreach (explode(',', urldecode($value)) as $title) {
-            $titles[] = explode('|', $title)[1];
-        }
-
-        return $titles;
     }
 
-    /**
+    protected function getUtilityForMode() : string
-     * @param array $tcaColumn
-     * @return array
-     */
-    protected function resolveInlineValue(array $tcaColumn)
     {
-        $titles = [];
+        if (TYPO3_MODE === 'BE') {
-
+            return BackendUtility::class;
-        foreach ($tcaColumn['children'] as $selected) {
-            $titles[] = $selected['recordTitle'];
         }
 
-        return $titles;
+        return FrontendUtility::class;
     }
 }

Classes/Domain/Index/TcaIndexer/TcaTableService.php

@@ -21,14 +21,21 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer;
  */
 
 use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
+use Codappix\SearchCore\Database\Doctrine\Join;
+use Codappix\SearchCore\Database\Doctrine\Where;
 use Codappix\SearchCore\Domain\Index\IndexingException;
+use Codappix\SearchCore\Domain\Index\TcaIndexer\InvalidArgumentException;
 use TYPO3\CMS\Backend\Utility\BackendUtility;
+use TYPO3\CMS\Core\Database\ConnectionPool;
+use TYPO3\CMS\Core\Database\Query\QueryBuilder;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Core\Utility\RootlineUtility;
+use TYPO3\CMS\Extbase\Object\ObjectManagerInterface;
 
 /**
  * Encapsulate logik related to TCA configuration.
  */
-class TcaTableService
+class TcaTableService implements TcaTableServiceInterface
 {
     /**
      * TCA for current table.
@@ -47,15 +54,20 @@ class TcaTableService
      */
     protected $configuration;
 
+    /**
+     * @var RelationResolver
+     */
+    protected $relationResolver;
+
     /**
      * @var \TYPO3\CMS\Core\Log\Logger
      */
     protected $logger;
 
     /**
-     * @var RelationResolver
+     * @var ObjectManagerInterface
      */
-    protected $relationResolver;
+    protected $objectManager;
 
     /**
      * Inject log manager to get concrete logger from it.
@@ -67,6 +79,14 @@ class TcaTableService
         $this->logger = $logManager->getLogger(__CLASS__);
     }
 
+    /**
+     * @param ObjectManagerInterface $objectManager
+     */
+    public function injectObjectManager(ObjectManagerInterface $objectManager)
+    {
+        $this->objectManager = $objectManager;
+    }
+
     /**
      * @param string $tableName
      * @param ConfigurationContainerInterface $configuration
@@ -89,28 +109,36 @@ class TcaTableService
         $this->relationResolver = $relationResolver;
     }
 
-    /**
+    public function getTableName() : string
-     * @return string
-     */
-    public function getTableName()
     {
         return $this->tableName;
     }
 
-    /**
+    public function getTableClause() : string
-     * @return string
-     */
-    public function getTableClause()
     {
-        return $this->tableName . ' LEFT JOIN pages on ' . $this->tableName . '.pid = pages.uid';
+        return $this->tableName;
+    }
+
+    public function getRecords(int $offset, int $limit) : array
+    {
+        $records = $this->getQuery()
+            ->setFirstResult($offset)
+            ->setMaxResults($limit)
+            ->execute()
+            ->fetchAll();
+
+        return $records ?: [];
+    }
+
+    public function getRecord(int $identifier) : array
+    {
+        $query = $this->getQuery();
+        $query = $query->andWhere($this->getTableName() . '.uid = ' . $identifier);
+        $record = $query->execute()->fetch();
+
+        return $record ?: [];
     }
 
-    /**
-     * Filter the given records by root line blacklist settings.
-     *
-     * @param array &$records
-     * @return void
-     */
     public function filterRecordsByRootLineBlacklist(array &$records)
     {
         $records = array_filter(
@@ -121,10 +149,6 @@ class TcaTableService
         );
     }
 
-    /**
-     * Adjust record accordingly to configuration.
-     * @param array &$record
-     */
     public function prepareRecord(array &$record)
     {
         $this->relationResolver->resolveRelationsForRecord($this, $record);
@@ -137,49 +161,39 @@ class TcaTableService
         }
     }
 
-    /**
+    protected function getWhereClause() : Where
-     * @return string
-     */
-    public function getWhereClause()
     {
-        $whereClause = '1=1'
+        $parameters = [];
-            . BackendUtility::BEenableFields($this->tableName)
+        $whereClause = $this->getSystemWhereClause();
-            . BackendUtility::deleteClause($this->tableName)
 
-            . BackendUtility::BEenableFields('pages')
+        $userDefinedWhere = $this->configuration->getIfExists(
-            . BackendUtility::deleteClause('pages')
+            'indexing.' . $this->getTableName() . '.additionalWhereClause'
-            . ' AND pages.no_search = 0'
+        );
-            ;
-
-        $userDefinedWhere = $this->configuration->getIfExists('indexer.tca.' . $this->tableName . '.additionalWhereClause');
         if (is_string($userDefinedWhere)) {
             $whereClause .= ' AND ' . $userDefinedWhere;
         }
 
-        if ($this->isBlacklistedRootLineConfigured()) {
+        if ($this->isBlackListedRootLineConfigured()) {
-            $whereClause .= ' AND pages.uid NOT IN ('
+            $parameters[':blacklistedRootLine'] = implode(',', $this->getBlackListedRootLine());
-                . implode(',', $this->getBlacklistedRootLine())
+            $whereClause .= ' AND pages.uid NOT IN (:blacklistedRootLine)'
-                . ')'
+                . ' AND pages.pid NOT IN (:blacklistedRootLine)';
-                . ' AND pages.pid NOT IN ('
-                . implode(',', $this->getBlacklistedRootLine())
-                . ')';
         }
 
         $this->logger->debug('Generated where clause.', [$this->tableName, $whereClause]);
-        return $whereClause;
+        return new Where($whereClause, $parameters);
     }
 
-    /**
+    protected function getFields() : array
-     * @return string
-     */
-    public function getFields()
     {
         $fields = array_merge(
             ['uid','pid'],
             array_filter(
                 array_keys($this->tca['columns']),
                 function ($columnName) {
-                    return !$this->isSystemField($columnName);
+                    return !$this->isSystemField($columnName)
+                        && !$this->isUserField($columnName)
+                        && !$this->isPassthroughField($columnName)
+                        ;
                 }
             )
         );
@@ -189,14 +203,42 @@ class TcaTableService
         }
 
         $this->logger->debug('Generated fields.', [$this->tableName, $fields]);
-        return implode(',', $fields);
+        return $fields;
+    }
+
+    protected function getJoins() : array
+    {
+        if ($this->tableName === 'pages') {
+            return [];
+        }
+
+        return [
+            new Join('pages', 'pages.uid = ' . $this->tableName . '.pid'),
+        ];
     }
 
     /**
-     * @param string
+     * Generate SQL for TYPO3 as a system, to make sure only available records
-     * @return bool
+     * are fetched.
      */
-    protected function isSystemField($columnName)
+    protected function getSystemWhereClause() : string
+    {
+        $whereClause = '1=1'
+            . BackendUtility::BEenableFields($this->tableName)
+            . BackendUtility::deleteClause($this->tableName)
+            . ' AND pages.no_search = 0'
+            ;
+
+        if ($this->tableName !== 'pages') {
+            $whereClause .= BackendUtility::BEenableFields('pages')
+                . BackendUtility::deleteClause('pages')
+            ;
+        }
+
+        return $whereClause;
+    }
+
+    protected function isSystemField(string $columnName) : bool
     {
         $systemFields = [
             // Versioning fields,
@@ -208,19 +250,28 @@ class TcaTableService
             $this->tca['ctrl']['cruser_id'],
             $this->tca['ctrl']['fe_cruser_id'],
             $this->tca['ctrl']['fe_crgroup_id'],
-            $this->tca['ctrl']['languageField'],
             $this->tca['ctrl']['origUid'],
         ];
 
         return in_array($columnName, $systemFields);
     }
 
+    protected function isUserField(string $columnName) : bool
+    {
+        $config = $this->getColumnConfig($columnName);
+        return isset($config['type']) && $config['type'] === 'user';
+    }
+
+    protected function isPassthroughField(string $columnName) : bool
+    {
+        $config = $this->getColumnConfig($columnName);
+        return isset($config['type']) && $config['type'] === 'passthrough';
+    }
+
     /**
-     * @param string $columnName
-     * @return array
      * @throws InvalidArgumentException
      */
-    public function getColumnConfig($columnName)
+    public function getColumnConfig(string $columnName) : array
     {
         if (!isset($this->tca['columns'][$columnName])) {
             throw new InvalidArgumentException(
@@ -236,26 +287,56 @@ class TcaTableService
      * Checks whether the given record was blacklisted by root line.
      * This can be configured by typoscript as whole root lines can be black listed.
      *
-     * NOTE: Does not support pages yet. We have to add a switch once we
+     * Also further TYPO3 mechanics are taken into account. Does a valid root
-     * support them to use uid instead.
+     * line exist, is page inside a recycler, is inherited start- endtime
-     *
+     * excluded, etc.
-     * @param array &$record
-     * @return bool
      */
-    protected function isRecordBlacklistedByRootline(array &$record)
+    protected function isRecordBlacklistedByRootline(array &$record) : bool
     {
-        // If no rootline exists, the record is on a unreachable page and therefore blacklisted.
+        $pageUid = $record['pid'];
-        $rootline = BackendUtility::BEgetRootLine($record['pid']);
+        if ($this->tableName === 'pages') {
-        if (!isset($rootline[0])) {
+            $pageUid = $record['uid'];
+        }
+
+        try {
+            $rootline = $this->objectManager->get(RootlineUtility::class, $pageUid)->get();
+        } catch (\RuntimeException $e) {
+            $this->logger->notice(
+                sprintf('Could not fetch rootline for page %u, because: %s', $pageUid, $e->getMessage()),
+                [$record, $e]
+            );
             return true;
         }
 
-        // Check configured black list if present.
+        foreach ($rootline as $pageInRootLine) {
-        if ($this->isBlackListedRootLineConfigured()) {
+            // Check configured black list if present.
-            foreach ($rootline as $pageInRootLine) {
+            if ($this->isBlackListedRootLineConfigured()
-                if (in_array($pageInRootLine['uid'], $this->getBlackListedRootLine())) {
+                && in_array($pageInRootLine['uid'], $this->getBlackListedRootLine())
-                    return true;
+            ) {
-                }
+                $this->logger->info(
+                    sprintf(
+                        'Record %u is black listed due to configured root line configuration of page %u.',
+                        $record['uid'],
+                        $pageInRootLine['uid']
+                    ),
+                    [$record, $pageInRootLine]
+                );
+                return true;
+            }
+
+            if ($pageInRootLine['extendToSubpages'] && (
+                ($pageInRootLine['endtime'] > 0 && $pageInRootLine['endtime'] <= time())
+                || ($pageInRootLine['starttime'] > 0 && $pageInRootLine['starttime'] >= time())
+            )) {
+                $this->logger->info(
+                    sprintf(
+                        'Record %u is black listed due to configured timing of parent page %u.',
+                        $record['uid'],
+                        $pageInRootLine['uid']
+                    ),
+                    [$record, $pageInRootLine]
+                );
+                return true;
             }
         }
 
@@ -264,12 +345,10 @@ class TcaTableService
 
     /**
      * Checks whether any page uids are black listed.
-     *
-     * @return bool
      */
-    protected function isBlackListedRootLineConfigured()
+    protected function isBlackListedRootLineConfigured() : bool
     {
-        return (bool) $this->configuration->getIfExists('indexer.tca.rootLineBlacklist');
+        return (bool) $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist');
     }
 
     /**
@@ -277,8 +356,33 @@ class TcaTableService
      *
      * @return array<Int>
      */
-    protected function getBlackListedRootLine()
+    protected function getBlackListedRootLine() : array
     {
-        return GeneralUtility::intExplode(',', $this->configuration->getIfExists('indexer.tca.rootLineBlacklist'));
+        return GeneralUtility::intExplode(
+            ',',
+            $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist')
+        );
+    }
+
+    public function getQuery() : QueryBuilder
+    {
+        $queryBuilder = $this->getDatabaseConnection()->getQueryBuilderForTable($this->getTableName());
+        $where = $this->getWhereClause();
+        $query = $queryBuilder->select(... $this->getFields())
+            ->from($this->getTableClause())
+            ->where($where->getStatement())
+            ->setParameters($where->getParameters());
+
+        foreach ($this->getJoins() as $join) {
+            $query->from($join->getTable());
+            $query->andWhere($join->getCondition());
+        }
+
+        return $query;
+    }
+
+    protected function getDatabaseConnection() : ConnectionPool
+    {
+        return GeneralUtility::makeInstance(ConnectionPool::class);
     }
 }

Classes/Domain/Index/TcaIndexer/TcaTableService76.php

@@ -0,0 +1,378 @@
+<?php
+namespace Codappix\SearchCore\Domain\Index\TcaIndexer;
+
+/*
+ * Copyright (C) 2016  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
+use Codappix\SearchCore\Domain\Index\IndexingException;
+use Codappix\SearchCore\Domain\Index\TcaIndexer\InvalidArgumentException;
+use TYPO3\CMS\Backend\Utility\BackendUtility;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Core\Utility\RootlineUtility;
+use TYPO3\CMS\Extbase\Object\ObjectManagerInterface;
+
+/**
+ * Encapsulate logik related to TCA configuration.
+ */
+class TcaTableService76 implements TcaTableServiceInterface
+{
+    /**
+     * TCA for current table.
+     * !REFERENCE! To save memory.
+     * @var array
+     */
+    protected $tca;
+
+    /**
+     * @var string
+     */
+    protected $tableName;
+
+    /**
+     * @var ConfigurationContainerInterface
+     */
+    protected $configuration;
+
+    /**
+     * @var RelationResolver
+     */
+    protected $relationResolver;
+
+    /**
+     * @var \TYPO3\CMS\Core\Log\Logger
+     */
+    protected $logger;
+
+    /**
+     * @var ObjectManagerInterface
+     */
+    protected $objectManager;
+
+    /**
+     * Inject log manager to get concrete logger from it.
+     *
+     * @param \TYPO3\CMS\Core\Log\LogManager $logManager
+     */
+    public function injectLogger(\TYPO3\CMS\Core\Log\LogManager $logManager)
+    {
+        $this->logger = $logManager->getLogger(__CLASS__);
+    }
+
+    /**
+     * @param ObjectManagerInterface $objectManager
+     */
+    public function injectObjectManager(ObjectManagerInterface $objectManager)
+    {
+        $this->objectManager = $objectManager;
+    }
+
+    /**
+     * @param string $tableName
+     * @param ConfigurationContainerInterface $configuration
+     */
+    public function __construct(
+        $tableName,
+        RelationResolver $relationResolver,
+        ConfigurationContainerInterface $configuration
+    ) {
+        if (!isset($GLOBALS['TCA'][$tableName])) {
+            throw new IndexingException(
+                'Table "' . $tableName . '" is not configured in TCA.',
+                IndexingException::CODE_UNKOWN_TCA_TABLE
+            );
+        }
+
+        $this->tableName = $tableName;
+        $this->tca = &$GLOBALS['TCA'][$this->tableName];
+        $this->configuration = $configuration;
+        $this->relationResolver = $relationResolver;
+    }
+
+    public function getTableName() : string
+    {
+        return $this->tableName;
+    }
+
+    public function getTableClause() : string
+    {
+        if ($this->tableName === 'pages') {
+            return $this->tableName;
+        }
+
+        return $this->tableName . ' LEFT JOIN pages on ' . $this->tableName . '.pid = pages.uid';
+    }
+
+    public function getRecords(int $offset, int $limit) : array
+    {
+        $records = $this->getConnection()->exec_SELECTgetRows(
+            $this->getFields(),
+            $this->getTableClause(),
+            $this->getWhereClause(),
+            '',
+            '',
+            (int) $offset . ',' . (int) $limit
+        );
+
+        return $records ?: [];
+    }
+
+    public function getRecord(int $identifier) : array
+    {
+        $record = $this->getConnection()->exec_SELECTgetSingleRow(
+            $this->getFields(),
+            $this->getTableClause(),
+            $this->getWhereClause()
+            . ' AND ' . $this->getTableName() . '.uid = ' . (int) $identifier
+        );
+
+        return $record ?: [];
+    }
+
+    public function filterRecordsByRootLineBlacklist(array &$records)
+    {
+        $records = array_filter(
+            $records,
+            function ($record) {
+                return ! $this->isRecordBlacklistedByRootline($record);
+            }
+        );
+    }
+
+    public function prepareRecord(array &$record)
+    {
+        $this->relationResolver->resolveRelationsForRecord($this, $record);
+
+        if (isset($record['uid']) && !isset($record['search_identifier'])) {
+            $record['search_identifier'] = $record['uid'];
+        }
+        if (isset($record[$this->tca['ctrl']['label']]) && !isset($record['search_title'])) {
+            $record['search_title'] = $record[$this->tca['ctrl']['label']];
+        }
+    }
+
+    public function getWhereClause() : string
+    {
+        $whereClause = '1=1'
+            . BackendUtility::BEenableFields($this->tableName)
+            . BackendUtility::deleteClause($this->tableName)
+            . ' AND pages.no_search = 0'
+            ;
+
+        if ($this->tableName !== 'pages') {
+            $whereClause .= BackendUtility::BEenableFields('pages')
+                . BackendUtility::deleteClause('pages')
+                ;
+        }
+
+        $userDefinedWhere = $this->configuration->getIfExists(
+            'indexing.' . $this->getTableName() . '.additionalWhereClause'
+        );
+        if (is_string($userDefinedWhere)) {
+            $whereClause .= ' AND ' . $userDefinedWhere;
+        }
+        if ($this->isBlacklistedRootLineConfigured()) {
+            $whereClause .= ' AND pages.uid NOT IN ('
+                . implode(',', $this->getBlacklistedRootLine())
+                . ')'
+                . ' AND pages.pid NOT IN ('
+                . implode(',', $this->getBlacklistedRootLine())
+                . ')';
+        }
+
+        $this->logger->debug('Generated where clause.', [$this->tableName, $whereClause]);
+        return $whereClause;
+    }
+
+    public function getFields() : string
+    {
+        $fields = array_merge(
+            ['uid','pid'],
+            array_filter(
+                array_keys($this->tca['columns']),
+                function ($columnName) {
+                    return !$this->isSystemField($columnName)
+                        && !$this->isUserField($columnName)
+                        && !$this->isPassthroughField($columnName)
+                        ;
+                }
+            )
+        );
+
+        foreach ($fields as $key => $field) {
+            $fields[$key] = $this->tableName . '.' . $field;
+        }
+
+        $this->logger->debug('Generated fields.', [$this->tableName, $fields]);
+        return implode(',', $fields);
+    }
+
+
+    /**
+     * Generate SQL for TYPO3 as a system, to make sure only available records
+     * are fetched.
+     */
+    protected function getSystemWhereClause() : string
+    {
+        $whereClause = '1=1'
+            . BackendUtility::BEenableFields($this->tableName)
+            . BackendUtility::deleteClause($this->tableName)
+            . ' AND pages.no_search = 0'
+            ;
+
+        if ($this->tableName !== 'pages') {
+            $whereClause .= BackendUtility::BEenableFields('pages')
+                . BackendUtility::deleteClause('pages')
+                ;
+        }
+
+        return $whereClause;
+    }
+
+    protected function isSystemField(string $columnName) : bool
+    {
+        $systemFields = [
+            // Versioning fields,
+            // https://docs.typo3.org/typo3cms/TCAReference/Reference/Ctrl/Index.html#versioningws
+            't3ver_oid', 't3ver_id', 't3ver_label', 't3ver_wsid',
+            't3ver_state', 't3ver_stage', 't3ver_count', 't3ver_tstamp',
+            't3ver_move_id', 't3ver_swapmode',
+            $this->tca['ctrl']['transOrigDiffSourceField'],
+            $this->tca['ctrl']['cruser_id'],
+            $this->tca['ctrl']['fe_cruser_id'],
+            $this->tca['ctrl']['fe_crgroup_id'],
+            $this->tca['ctrl']['origUid'],
+        ];
+
+        return in_array($columnName, $systemFields);
+    }
+
+    protected function isUserField(string $columnName) : bool
+    {
+        $config = $this->getColumnConfig($columnName);
+        return isset($config['type']) && $config['type'] === 'user';
+    }
+
+    protected function isPassthroughField(string $columnName) : bool
+    {
+        $config = $this->getColumnConfig($columnName);
+        return isset($config['type']) && $config['type'] === 'passthrough';
+    }
+
+    /**
+     * @throws InvalidArgumentException
+     */
+    public function getColumnConfig(string $columnName) : array
+    {
+        if (!isset($this->tca['columns'][$columnName])) {
+            throw new InvalidArgumentException(
+                'Column does not exist.',
+                InvalidArgumentException::COLUMN_DOES_NOT_EXIST
+            );
+        }
+
+        return $this->tca['columns'][$columnName]['config'];
+    }
+
+    /**
+     * Checks whether the given record was blacklisted by root line.
+     * This can be configured by typoscript as whole root lines can be black listed.
+     *
+     * Also further TYPO3 mechanics are taken into account. Does a valid root
+     * line exist, is page inside a recycler, is inherited start- endtime
+     * excluded, etc.
+     */
+    protected function isRecordBlacklistedByRootline(array &$record) : bool
+    {
+        $pageUid = $record['pid'];
+        if ($this->tableName === 'pages') {
+            $pageUid = $record['uid'];
+        }
+
+        try {
+            $rootline = $this->objectManager->get(RootlineUtility::class, $pageUid)->get();
+        } catch (\RuntimeException $e) {
+            $this->logger->notice(
+                sprintf('Could not fetch rootline for page %u, because: %s', $pageUid, $e->getMessage()),
+                [$record, $e]
+            );
+            return true;
+        }
+
+        foreach ($rootline as $pageInRootLine) {
+            // Check configured black list if present.
+            if ($this->isBlackListedRootLineConfigured()
+                && in_array($pageInRootLine['uid'], $this->getBlackListedRootLine())
+            ) {
+                $this->logger->info(
+                    sprintf(
+                        'Record %u is black listed due to configured root line configuration of page %u.',
+                        $record['uid'],
+                        $pageInRootLine['uid']
+                    ),
+                    [$record, $pageInRootLine]
+                );
+                return true;
+            }
+
+            if ($pageInRootLine['extendToSubpages'] && (
+                ($pageInRootLine['endtime'] > 0 && $pageInRootLine['endtime'] <= time())
+                || ($pageInRootLine['starttime'] > 0 && $pageInRootLine['starttime'] >= time())
+            )) {
+                $this->logger->info(
+                    sprintf(
+                        'Record %u is black listed due to configured timing of parent page %u.',
+                        $record['uid'],
+                        $pageInRootLine['uid']
+                    ),
+                    [$record, $pageInRootLine]
+                );
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * Checks whether any page uids are black listed.
+     */
+    protected function isBlackListedRootLineConfigured() : bool
+    {
+        return (bool) $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist');
+    }
+
+    /**
+     * Get the list of black listed root line page uids.
+     *
+     * @return array<Int>
+     */
+    protected function getBlackListedRootLine() : array
+    {
+        return GeneralUtility::intExplode(
+            ',',
+            $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist')
+        );
+    }
+
+    protected function getConnection() : \TYPO3\CMS\Core\Database\DatabaseConnection
+    {
+        return $GLOBALS['TYPO3_DB'];
+    }
+}

Classes/Domain/Index/TcaIndexer/TcaTableServiceInterface.php

@@ -0,0 +1,44 @@
+<?php
+namespace Codappix\SearchCore\Domain\Index\TcaIndexer;
+
+/*
+ * Copyright (C) 2016  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+interface TcaTableServiceInterface
+{
+    public function getTableName() : string;
+
+    public function getTableClause() : string;
+
+    /**
+     * Filter the given records by root line blacklist settings.
+     */
+    public function filterRecordsByRootLineBlacklist(array &$records);
+
+    public function prepareRecord(array &$record);
+
+    /**
+     * @throws InvalidArgumentException
+     */
+    public function getColumnConfig(string $columnName) : array;
+
+    public function getRecords(int $offset, int $limit) : array;
+
+    public function getRecord(int $identifier) : array;
+}

Classes/Domain/Model/FacetRequest.php

@@ -0,0 +1,56 @@
+<?php
+namespace Codappix\SearchCore\Domain\Model;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\SearchCore\Connection\FacetRequestInterface;
+
+class FacetRequest implements FacetRequestInterface
+{
+    /**
+     * @var string
+     */
+    protected $identifier = '';
+
+    /**
+     * @var array
+     */
+    protected $config = [];
+
+    /**
+     * As the facets come from configuration this might be a good idea to help
+     * integrators find issues.
+     */
+    public function __construct(string $identifier, array $config)
+    {
+        $this->identifier = $identifier;
+        $this->config = $config;
+    }
+
+    public function getIdentifier() : string
+    {
+        return $this->identifier;
+    }
+
+    public function getConfig() : array
+    {
+        return $this->config;
+    }
+}

Classes/Domain/Model/QueryResultInterfaceStub.php

@@ -0,0 +1,61 @@
+<?php
+namespace Codappix\SearchCore\Domain\Model;
+
+/*
+ * Copyright (C) 2018  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * As we have to stay compatible with QueryResultInterface
+ * of extbase but can and need not to provide all methods,
+ * this stub will provde the non implemented methods to
+ * keep real implementations clean.
+ */
+trait QueryResultInterfaceStub
+{
+    public function getFirst()
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502195121);
+    }
+
+    public function toArray()
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502195135);
+    }
+
+    public function offsetExists($offset)
+    {
+        // Return false to allow Fluid to use appropriate getter methods.
+        return false;
+    }
+
+    public function offsetGet($offset)
+    {
+        throw new \BadMethodCallException('Use getter to fetch properties.', 1502196933);
+    }
+
+    public function offsetSet($offset, $value)
+    {
+        throw new \BadMethodCallException('You are not allowed to modify the result.', 1502196934);
+    }
+
+    public function offsetUnset($offset)
+    {
+        throw new \BadMethodCallException('You are not allowed to modify the result.', 1502196936);
+    }
+}

Classes/Domain/Model/ResultItem.php

@@ -0,0 +1,61 @@
+<?php
+namespace Codappix\SearchCore\Domain\Model;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\SearchCore\Connection\ResultItemInterface;
+
+class ResultItem implements ResultItemInterface
+{
+    /**
+     * @var array
+     */
+    protected $data = [];
+
+    public function __construct(array $result)
+    {
+        $this->data = $result;
+    }
+
+    public function getPlainData() : array
+    {
+        return $this->data;
+    }
+
+    public function offsetExists($offset)
+    {
+        return isset($this->data[$offset]);
+    }
+
+    public function offsetGet($offset)
+    {
+        return $this->data[$offset];
+    }
+
+    public function offsetSet($offset, $value)
+    {
+        throw new \BadMethodCallException('It\'s not possible to change the search result.', 1499179077);
+    }
+
+    public function offsetUnset($offset)
+    {
+        throw new \BadMethodCallException('It\'s not possible to change the search result.', 1499179077);
+    }
+}

Classes/Domain/Model/SearchRequest.php

@@ -20,7 +20,10 @@ namespace Codappix\SearchCore\Domain\Model;
  * 02110-1301, USA.
  */
 
+use Codappix\SearchCore\Connection\ConnectionInterface;
+use Codappix\SearchCore\Connection\FacetRequestInterface;
 use Codappix\SearchCore\Connection\SearchRequestInterface;
+use Codappix\SearchCore\Domain\Search\SearchService;
 
 /**
  * Represents a search request used to process an actual search.
@@ -32,29 +35,263 @@ class SearchRequest implements SearchRequestInterface
      *
      * @var string
      */
-    protected $query;
+    protected $query = '';
+
+    /**
+     * @var array
+     */
+    protected $filter = [];
+
+    /**
+     * @var array
+     */
+    protected $facets = [];
+
+    /**
+     * @var int
+     */
+    protected $offset = 0;
+
+    /**
+     * @var int
+     */
+    protected $limit = 10;
+
+    /**
+     * Used for QueryInterface implementation to allow execute method to work.
+     *
+     * @var ConnectionInterface
+     */
+    protected $connection = null;
+
+    /**
+     * @var SearchService
+     */
+    protected $searchService = null;
 
     /**
      * @param string $query
      */
-    public function __construct($query)
+    public function __construct(string $query = '')
     {
         $this->query = $query;
     }
 
-    /**
+    public function getQuery() : string
-     * @return string
+    {
-     */
+        return $this->query;
-    public function getQuery()
+    }
+
+    public function getSearchTerm() : string
     {
         return $this->query;
     }
 
     /**
-     * @return string
+     * @param array $filter
      */
-    public function getSearchTerm()
+    public function setFilter(array $filter)
     {
-        return $this->query;
+        $filter = \TYPO3\CMS\Core\Utility\ArrayUtility::removeArrayEntryByValue($filter, '');
+        $this->filter = \TYPO3\CMS\Extbase\Utility\ArrayUtility::removeEmptyElementsRecursively($filter);
+    }
+
+    public function hasFilter() : bool
+    {
+        return count($this->filter) > 0;
+    }
+
+    public function getFilter() : array
+    {
+        return $this->filter;
+    }
+
+    /**
+     * Add a facet to gather in this search request.
+     */
+    public function addFacet(FacetRequestInterface $facet)
+    {
+        $this->facets[$facet->getIdentifier()] = $facet;
+    }
+
+    /**
+     * Returns all configured facets to fetch in this search request.
+     */
+    public function getFacets() : array
+    {
+        return $this->facets;
+    }
+
+    /**
+     * Define connection to use for this request.
+     * Necessary to allow implementation of execute for interface.
+     */
+    public function setConnection(ConnectionInterface $connection)
+    {
+        $this->connection = $connection;
+    }
+
+    public function setSearchService(SearchService $searchService)
+    {
+        $this->searchService = $searchService;
+    }
+
+    // Extbase QueryInterface
+    // Current implementation covers only paginate widget support.
+    public function execute($returnRawQueryResult = false)
+    {
+        if (! ($this->connection instanceof ConnectionInterface)) {
+            throw new \InvalidArgumentException(
+                'Connection was not set before, therefore execute can not work. Use `setConnection` before.',
+                1502197732
+            );
+        }
+        if (! ($this->searchService instanceof SearchService)) {
+            throw new \InvalidArgumentException(
+                'SearchService was not set before, therefore execute can not work. Use `setSearchService` before.',
+                1520325175
+            );
+        }
+
+        return $this->searchService->processResult($this->connection->search($this));
+    }
+
+    public function setLimit($limit)
+    {
+        $this->limit = (int) $limit;
+
+        return $this;
+    }
+
+    public function setOffset($offset)
+    {
+        $this->offset = (int) $offset;
+
+        return $this;
+    }
+
+    public function getLimit()
+    {
+        return $this->limit;
+    }
+
+    public function getOffset()
+    {
+        return $this->offset;
+    }
+
+    public function getSource()
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196146);
+    }
+
+    public function setOrderings(array $orderings)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196163);
+    }
+
+    public function matching($constraint)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196197);
+    }
+
+    public function logicalAnd($constraint1)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196166);
+    }
+
+    public function logicalOr($constraint1)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196198);
+    }
+
+    public function logicalNot(\TYPO3\CMS\Extbase\Persistence\Generic\Qom\ConstraintInterface $constraint)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196166);
+    }
+
+    public function equals($propertyName, $operand, $caseSensitive = true)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196199);
+    }
+
+    public function like($propertyName, $operand, $caseSensitive = true)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196167);
+    }
+
+    public function contains($propertyName, $operand)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196200);
+    }
+
+    public function in($propertyName, $operand)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196167);
+    }
+
+    public function lessThan($propertyName, $operand)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196201);
+    }
+
+    public function lessThanOrEqual($propertyName, $operand)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196168);
+    }
+
+    public function greaterThan($propertyName, $operand)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196202);
+    }
+
+    public function greaterThanOrEqual($propertyName, $operand)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196168);
+    }
+
+    public function getType()
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196203);
+    }
+
+    public function setQuerySettings(\TYPO3\CMS\Extbase\Persistence\Generic\QuerySettingsInterface $querySettings)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196168);
+    }
+
+    public function getQuerySettings()
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196205);
+    }
+
+    public function count()
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196169);
+    }
+
+    public function getOrderings()
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196206);
+    }
+
+    public function getConstraint()
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196171);
+    }
+
+    public function isEmpty($propertyName)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196207);
+    }
+
+    public function setSource(\TYPO3\CMS\Extbase\Persistence\Generic\Qom\SourceInterface $source)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196172);
+    }
+
+    public function getStatement()
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196208);
     }
 }

Classes/Domain/Model/SearchResult.php

@@ -0,0 +1,129 @@
+<?php
+namespace Codappix\SearchCore\Domain\Model;
+
+/*
+ * Copyright (C) 2016  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\SearchCore\Connection\ResultItemInterface;
+use Codappix\SearchCore\Connection\SearchResultInterface;
+use Codappix\SearchCore\Domain\Model\QueryResultInterfaceStub;
+
+/**
+ * Generic model for mapping a concrete search result from a connection.
+ */
+class SearchResult implements SearchResultInterface
+{
+    use QueryResultInterfaceStub;
+
+    /**
+     * @var SearchResultInterface
+     */
+    protected $originalSearchResult;
+
+    /**
+     * @var array
+     */
+    protected $resultItems = [];
+
+    /**
+     * @var array
+     */
+    protected $results = [];
+
+    /**
+     * For Iterator interface.
+     *
+     * @var int
+     */
+    protected $position = 0;
+
+    public function __construct(SearchResultInterface $originalSearchResult, array $resultItems)
+    {
+        $this->originalSearchResult = $originalSearchResult;
+        $this->resultItems = $resultItems;
+    }
+
+    /**
+     * @return array<ResultItemInterface>
+     */
+    public function getResults() : array
+    {
+        $this->initResults();
+
+        return $this->results;
+    }
+
+    protected function initResults()
+    {
+        if ($this->results !== []) {
+            return;
+        }
+
+        foreach ($this->resultItems as $item) {
+            $this->results[] = new ResultItem($item);
+        }
+    }
+
+    public function getFacets() : array
+    {
+        return $this->originalSearchResult->getFacets();
+    }
+
+    public function getCurrentCount() : int
+    {
+        return $this->originalSearchResult->getCurrentCount();
+    }
+
+    public function count()
+    {
+        return $this->originalSearchResult->count();
+    }
+
+    public function current()
+    {
+        return $this->getResults()[$this->position];
+    }
+
+    public function next()
+    {
+        ++$this->position;
+
+        return $this->current();
+    }
+
+    public function key()
+    {
+        return $this->position;
+    }
+
+    public function valid()
+    {
+        return isset($this->getResults()[$this->position]);
+    }
+
+    public function rewind()
+    {
+        $this->position = 0;
+    }
+
+    public function getQuery()
+    {
+        return $this->originalSearchResult->getQuery();
+    }
+}

Classes/Domain/Search/QueryFactory.php

@@ -0,0 +1,270 @@
+<?php
+namespace Codappix\SearchCore\Domain\Search;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
+use Codappix\SearchCore\Configuration\ConfigurationUtility;
+use Codappix\SearchCore\Configuration\InvalidArgumentException;
+use Codappix\SearchCore\Connection\SearchRequestInterface;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Extbase\Utility\ArrayUtility;
+
+class QueryFactory
+{
+    /**
+     * @var \TYPO3\CMS\Core\Log\Logger
+     */
+    protected $logger;
+
+    /**
+     * @var ConfigurationContainerInterface
+     */
+    protected $configuration;
+
+    /**
+     * @var ConfigurationUtility
+     */
+    protected $configurationUtility;
+
+    public function __construct(
+        \TYPO3\CMS\Core\Log\LogManager $logManager,
+        ConfigurationContainerInterface $configuration,
+        ConfigurationUtility $configurationUtility
+    ) {
+        $this->logger = $logManager->getLogger(__CLASS__);
+        $this->configuration = $configuration;
+        $this->configurationUtility = $configurationUtility;
+    }
+
+    /**
+     * TODO: This is not in scope Elasticsearch, therefore it should not return
+     * \Elastica\Query, but decide to use a more specific QueryFactory like
+     * ElasticaQueryFactory, once the second query is added?
+     */
+    public function create(SearchRequestInterface $searchRequest) : \Elastica\Query
+    {
+        return $this->createElasticaQuery($searchRequest);
+    }
+
+    protected function createElasticaQuery(SearchRequestInterface $searchRequest) : \Elastica\Query
+    {
+        $query = [];
+        $this->addSize($searchRequest, $query);
+        $this->addSearch($searchRequest, $query);
+        $this->addBoosts($searchRequest, $query);
+        $this->addFilter($searchRequest, $query);
+        $this->addFacets($searchRequest, $query);
+        $this->addFields($searchRequest, $query);
+        $this->addSort($searchRequest, $query);
+
+        // Use last, as it might change structure of query.
+        // Better approach would be something like DQL to generate query and build result in the end.
+        $this->addFactorBoost($query);
+
+        $this->logger->debug('Generated elasticsearch query.', [$query]);
+        return new \Elastica\Query($query);
+    }
+
+    protected function addSize(SearchRequestInterface $searchRequest, array &$query)
+    {
+        $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [
+            'from' => $searchRequest->getOffset(),
+            'size' => $searchRequest->getLimit(),
+        ]);
+    }
+
+    protected function addSearch(SearchRequestInterface $searchRequest, array &$query)
+    {
+        if (trim($searchRequest->getSearchTerm()) === '') {
+            return;
+        }
+
+        $matchExpression = [
+            'type' => 'most_fields',
+            'query' => $searchRequest->getSearchTerm(),
+            'fields' => GeneralUtility::trimExplode(',', $this->configuration->get('searching.fields.query')),
+        ];
+
+        $minimumShouldMatch = $this->configuration->getIfExists('searching.minimumShouldMatch');
+        if ($minimumShouldMatch) {
+            $matchExpression['minimum_should_match'] = $minimumShouldMatch;
+        }
+
+        $query = ArrayUtility::setValueByPath($query, 'query.bool.must.0.multi_match', $matchExpression);
+    }
+
+    protected function addBoosts(SearchRequestInterface $searchRequest, array &$query)
+    {
+        try {
+            $fields = $this->configuration->get('searching.boost');
+        } catch (InvalidArgumentException $e) {
+            return;
+        }
+
+        if (trim($searchRequest->getSearchTerm()) === '') {
+            return;
+        }
+
+        $boostQueryParts = [];
+
+        foreach ($fields as $fieldName => $boostValue) {
+            $boostQueryParts[] = [
+                'match' => [
+                    $fieldName => [
+                        'query' => $searchRequest->getSearchTerm(),
+                        'boost' => $boostValue,
+                    ],
+                ],
+            ];
+        }
+
+        if (!empty($boostQueryParts)) {
+            $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [
+                'query' => [
+                    'bool' => [
+                        'should' => $boostQueryParts,
+                    ],
+                ],
+            ]);
+        }
+    }
+
+    protected function addFactorBoost(array &$query)
+    {
+        try {
+            $query['query'] = [
+                'function_score' => [
+                    'query' => $query['query'],
+                    'field_value_factor' => $this->configuration->get('searching.fieldValueFactor'),
+                ],
+            ];
+        } catch (InvalidArgumentException $e) {
+            return;
+        }
+    }
+
+    protected function addFields(SearchRequestInterface $searchRequest, array &$query)
+    {
+        try {
+            $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [
+                'stored_fields' => GeneralUtility::trimExplode(
+                    ',',
+                    $this->configuration->get('searching.fields.stored_fields'),
+                    true
+                ),
+            ]);
+        } catch (InvalidArgumentException $e) {
+            // Nothing configured
+        }
+
+        try {
+            $scriptFields = $this->configuration->get('searching.fields.script_fields');
+            $scriptFields = $this->configurationUtility->replaceArrayValuesWithRequestContent(
+                $searchRequest,
+                $scriptFields
+            );
+            $scriptFields = $this->configurationUtility->filterByCondition($scriptFields);
+            if ($scriptFields !== []) {
+                $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['script_fields' => $scriptFields]);
+            }
+        } catch (InvalidArgumentException $e) {
+            // Nothing configured
+        }
+    }
+
+    protected function addSort(SearchRequestInterface $searchRequest, array &$query)
+    {
+        $sorting = $this->configuration->getIfExists('searching.sort') ?: [];
+        $sorting = $this->configurationUtility->replaceArrayValuesWithRequestContent($searchRequest, $sorting);
+        $sorting = $this->configurationUtility->filterByCondition($sorting);
+        if ($sorting !== []) {
+            $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['sort' => $sorting]);
+        }
+    }
+
+    protected function addFilter(SearchRequestInterface $searchRequest, array &$query)
+    {
+        if (! $searchRequest->hasFilter()) {
+            return;
+        }
+
+        $filter = [];
+        foreach ($searchRequest->getFilter() as $name => $value) {
+            $filter[] = $this->buildFilter(
+                $name,
+                $value,
+                $this->configuration->getIfExists('searching.mapping.filter.' . $name) ?: []
+            );
+        }
+
+        $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [
+            'query' => [
+                'bool' => [
+                    'filter' => $filter,
+                ],
+            ],
+        ]);
+    }
+
+    protected function buildFilter(string $name, $value, array $config) : array
+    {
+        if ($config === []) {
+            return [
+                'term' => [
+                    $name => $value,
+                ],
+            ];
+        }
+
+        $filter = [];
+
+        if (isset($config['fields'])) {
+            foreach ($config['fields'] as $elasticField => $inputField) {
+                $filter[$elasticField] = $value[$inputField];
+            }
+        }
+
+        if (isset($config['raw'])) {
+            $filter = array_merge($config['raw'], $filter);
+        }
+
+        if ($config['type'] === 'range') {
+            return [
+                'range' => [
+                    $config['field'] => $filter,
+                ],
+            ];
+        }
+
+        return [$config['field'] => $filter];
+    }
+
+    protected function addFacets(SearchRequestInterface $searchRequest, array &$query)
+    {
+        foreach ($searchRequest->getFacets() as $facet) {
+            $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [
+                'aggs' => [
+                    $facet->getIdentifier() => $facet->getConfig(),
+                ],
+            ]);
+        }
+    }
+}

Classes/Domain/Search/SearchService.php

@@ -20,10 +20,16 @@ namespace Codappix\SearchCore\Domain\Search;
  * 02110-1301, USA.
  */
 
+use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
+use Codappix\SearchCore\Configuration\InvalidArgumentException;
 use Codappix\SearchCore\Connection\ConnectionInterface;
 use Codappix\SearchCore\Connection\SearchRequestInterface;
 use Codappix\SearchCore\Connection\SearchResultInterface;
-use Codappix\SearchCore\Domain\Model\SearchRequest;
+use Codappix\SearchCore\DataProcessing\Service as DataProcessorService;
+use Codappix\SearchCore\Domain\Model\FacetRequest;
+use Codappix\SearchCore\Domain\Model\SearchResult;
+use TYPO3\CMS\Core\Utility\ArrayUtility;
+use TYPO3\CMS\Extbase\Object\ObjectManagerInterface;
 
 /**
  * Service to process a search request.
@@ -36,19 +42,124 @@ class SearchService
     protected $connection;
 
     /**
-     * @param ConnectionInterface $connection
+     * @var ConfigurationContainerInterface
      */
-    public function __construct(ConnectionInterface $connection)
+    protected $configuration;
-    {
+
+    /**
+     * @var ObjectManagerInterface
+     */
+    protected $objectManager;
+
+    /**
+     * @var DataProcessorService
+     */
+    protected $dataProcessorService;
+
+    /**
+     * @param ConnectionInterface $connection
+     * @param ConfigurationContainerInterface $configuration
+     * @param ObjectManagerInterface $objectManager
+     * @param DataProcessorService $dataProcessorService
+     */
+    public function __construct(
+        ConnectionInterface $connection,
+        ConfigurationContainerInterface $configuration,
+        ObjectManagerInterface $objectManager,
+        DataProcessorService $dataProcessorService
+    ) {
         $this->connection = $connection;
+        $this->configuration = $configuration;
+        $this->objectManager = $objectManager;
+        $this->dataProcessorService = $dataProcessorService;
+    }
+
+    public function search(SearchRequestInterface $searchRequest) : SearchResultInterface
+    {
+        $this->addSize($searchRequest);
+        $this->addConfiguredFacets($searchRequest);
+        $this->addConfiguredFilters($searchRequest);
+
+        // Add connection to request to enable paginate widget support
+        $searchRequest->setConnection($this->connection);
+        $searchRequest->setSearchService($this);
+
+        return $this->processResult($this->connection->search($searchRequest));
     }
 
     /**
-     * @param SearchRequestInterface $searchRequest
+     * Add configured size of search result items to request.
-     * @return SearchResultInterface
      */
-    public function search(SearchRequestInterface $searchRequest)
+    protected function addSize(SearchRequestInterface $searchRequest)
     {
-        return $this->connection->search($searchRequest);
+        $searchRequest->setLimit(
+            $this->configuration->getIfExists('searching.size') ?: 10
+        );
+    }
+
+    /**
+     * Add facets from configuration to request.
+     */
+    protected function addConfiguredFacets(SearchRequestInterface $searchRequest)
+    {
+        $facetsConfig = $this->configuration->getIfExists('searching.facets');
+        if ($facetsConfig === null) {
+            return;
+        }
+
+        foreach ($facetsConfig as $identifier => $facetConfig) {
+            $searchRequest->addFacet($this->objectManager->get(
+                FacetRequest::class,
+                $identifier,
+                $facetConfig
+            ));
+        }
+    }
+
+    /**
+     * Add filters from configuration, e.g. flexform or TypoScript.
+     */
+    protected function addConfiguredFilters(SearchRequestInterface $searchRequest)
+    {
+        try {
+            $filter = $searchRequest->getFilter();
+
+            ArrayUtility::mergeRecursiveWithOverrule(
+                $filter,
+                $this->configuration->get('searching.filter'),
+                true,
+                false
+            );
+
+            $searchRequest->setFilter($filter);
+        } catch (InvalidArgumentException $e) {
+            // Nothing todo, no filter configured.
+        }
+    }
+
+    /**
+     * Processes the result, e.g. applies configured data processing to result.
+     */
+    public function processResult(SearchResultInterface $searchResult) : SearchResultInterface
+    {
+        try {
+            $newSearchResultItems = [];
+            foreach ($this->configuration->get('searching.dataProcessing') as $configuration) {
+                foreach ($searchResult as $resultItem) {
+                    $newSearchResultItems[] = $this->dataProcessorService->executeDataProcessor(
+                        $configuration,
+                        $resultItem->getPlainData()
+                    );
+                }
+            }
+
+            return $this->objectManager->get(
+                SearchResult::class,
+                $searchResult,
+                $newSearchResultItems
+            );
+        } catch (InvalidArgumentException $e) {
+            return $searchResult;
+        }
     }
 }

Classes/Domain/Service/DataHandler.php

@@ -21,8 +21,10 @@ namespace Codappix\SearchCore\Domain\Service;
  */
 
 use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
+use Codappix\SearchCore\Domain\Index\IndexerFactory;
+use Codappix\SearchCore\Domain\Index\IndexerInterface;
+use Codappix\SearchCore\Domain\Index\NoMatchingIndexerException;
 use TYPO3\CMS\Core\SingletonInterface as Singleton;
-use TYPO3\CMS\Core\Utility\GeneralUtility;
 
 /**
  * Handles all data related things like updates, deletes and inserts.
@@ -46,8 +48,7 @@ class DataHandler implements Singleton
     protected $connection;
 
     /**
-     * @var \Codappix\SearchCore\Domain\Index\IndexerFactory
+     * @var IndexerFactory
-     * @inject
      */
     protected $indexerFactory;
 
@@ -73,48 +74,41 @@ class DataHandler implements Singleton
 
     /**
      * @param ConfigurationContainerInterface $configuration
+     * @param IndexerFactory $indexerFactory
      */
-    public function __construct(ConfigurationContainerInterface $configuration)
+    public function __construct(ConfigurationContainerInterface $configuration, IndexerFactory $indexerFactory)
     {
         $this->configuration = $configuration;
+        $this->indexerFactory = $indexerFactory;
     }
 
-    /**
+    public function update(string $table, array $record)
-     * Get all tables that are allowed for indexing.
-     *
-     * @return array<String>
-     */
-    public function getAllowedTablesForIndexing()
-    {
-        return GeneralUtility::trimExplode(',', $this->configuration->get('indexer.tca.allowedTables'));
-    }
-
-    /**
-     * @param string $table
-     * @param array $record
-     */
-    public function add($table, array $record)
-    {
-        $this->logger->debug('Record received for add.', [$table, $record]);
-        $this->indexerFactory->getIndexer($table)->indexDocument($record['uid']);
-    }
-
-    /**
-     * @param string $table
-     */
-    public function update($table, array $record)
     {
         $this->logger->debug('Record received for update.', [$table, $record]);
-        $this->indexerFactory->getIndexer($table)->indexDocument($record['uid']);
+        $this->getIndexer($table)->indexDocument($record['uid']);
     }
 
-    /**
+    public function delete(string $table, string $identifier)
-     * @param string $table
-     * @param int $identifier
-     */
-    public function delete($table, $identifier)
     {
         $this->logger->debug('Record received for delete.', [$table, $identifier]);
         $this->connection->deleteDocument($table, $identifier);
     }
+
+    /**
+     * @throws NoMatchingIndexerException
+     */
+    protected function getIndexer(string $table) : IndexerInterface
+    {
+        return $this->indexerFactory->getIndexer($table);
+    }
+
+    public function supportsTable(string $table) : bool
+    {
+        try {
+            $this->getIndexer($table);
+            return true;
+        } catch (NoMatchingIndexerException $e) {
+            return false;
+        }
+    }
 }

Classes/Hook/DataHandler.php

@@ -48,99 +48,99 @@ class DataHandler implements Singleton
     /**
      * Dependency injection as TYPO3 doesn't provide it on it's own.
      * Still you can submit your own dataHandler.
-     *
-     * @param OwnDataHandler $dataHandler
-     * @param Logger $logger
      */
     public function __construct(OwnDataHandler $dataHandler = null, Logger $logger = null)
     {
-        $this->dataHandler = $dataHandler;
+        if ($dataHandler === null) {
-        if ($this->dataHandler === null) {
             try {
-                $this->dataHandler = GeneralUtility::makeInstance(ObjectManager::class)
+                $dataHandler = GeneralUtility::makeInstance(ObjectManager::class)
                     ->get(OwnDataHandler::class);
             } catch (NoConfigurationException $e) {
                 // We have no configuration. That's fine, hooks will not be
                 // executed due to check for existing DataHandler.
             }
         }
+        $this->dataHandler = $dataHandler;
 
-        $this->logger = $logger;
+        if ($logger === null) {
-        if ($this->logger === null) {
+            $logger = GeneralUtility::makeInstance(LogManager::class)
-            $this->logger = GeneralUtility::makeInstance(LogManager::class)
                 ->getLogger(__CLASS__);
         }
+        $this->logger = $logger;
     }
 
     /**
      * Called by CoreDataHandler on deletion of records.
-     *
-     * @param string $table
-     * @param int $uid
-     *
-     * @return bool False if hook was not processed.
      */
-    public function processCmdmap_deleteAction($table, $uid)
+    public function processCmdmap_deleteAction(string $table, int $uid) : bool
     {
         if (! $this->shouldProcessHookForTable($table)) {
             $this->logger->debug('Delete not processed.', [$table, $uid]);
             return false;
         }
 
-        $this->dataHandler->delete($table, $uid);
+        $this->dataHandler->delete($table, (string) $uid);
         return true;
     }
 
-    /**
+    public function processDatamap_afterAllOperations(CoreDataHandler $dataHandler)
-     * Called by CoreDataHandler on database operations, e.g. if new records were created or records were updated.
+    {
-     *
+        foreach ($dataHandler->datamap as $table => $record) {
-     * @param string $status
+            $uid = key($record);
-     * @param string $table
+            $fieldData = current($record);
-     * @param string|int $uid
+
-     * @param array $fieldArray
+            if (isset($fieldData['uid'])) {
-     * @param CoreDataHandler $dataHandler
+                $uid = $fieldData['uid'];
-     *
+            } elseif (isset($dataHandler->substNEWwithIDs[$uid])) {
-     * @return bool False if hook was not processed.
+                $uid = $dataHandler->substNEWwithIDs[$uid];
-     */
+            }
-    public function processDatamap_afterDatabaseOperations($status, $table, $uid, array $fieldArray, CoreDataHandler $dataHandler)
+
+            $this->processRecord($table, $uid);
+        }
+    }
+
+    public function clearCachePostProc(array $parameters, CoreDataHandler $dataHandler)
+    {
+        $pageUid = 0;
+
+        // If editor uses "small page blizzard"
+        if (isset($parameters['cacheCmd']) && is_numeric($parameters['cacheCmd'])) {
+            $pageUid = $parameters['cacheCmd'];
+        }
+        // If records were changed
+        if (isset($parameters['uid_page']) && is_numeric($parameters['uid_page'])) {
+            $pageUid = $parameters['uid_page'];
+        }
+
+        if ($pageUid > 0) {
+            $this->processRecord('pages', (int) $pageUid);
+        }
+    }
+
+    protected function processRecord(string $table, int $uid) : bool
     {
         if (! $this->shouldProcessHookForTable($table)) {
-            $this->logger->debug('Database update not processed.', [$table, $uid]);
+            $this->logger->debug('Indexing of record not processed.', [$table, $uid]);
             return false;
         }
 
-        if ($status === 'new') {
+        $record = $this->getRecord($table, $uid);
-            $fieldArray['uid'] = $dataHandler->substNEWwithIDs[$uid];
+        if ($record !== null) {
-            $this->dataHandler->add($table, $fieldArray);
+            $this->dataHandler->update($table, $record);
             return true;
         }
 
-        if ($status === 'update') {
+        $this->logger->debug('Indexing of record not processed, as he was not found in Database.', [$table, $uid]);
-            $record = $this->getRecord($table, $uid);
-            if ($record !== null) {
-                $this->dataHandler->update($table, $record);
-            }
-            return true;
-        }
-
-        $this->logger->debug(
-            'Database update not processed, cause status is unhandled.',
-            [$status, $table, $uid, $fieldArray]
-        );
         return false;
     }
 
-    /**
+    protected function shouldProcessHookForTable(string $table) : bool
-     * @param string $table
-     * @return bool
-     */
-    protected function shouldProcessHookForTable($table)
     {
         if ($this->dataHandler === null) {
             $this->logger->debug('Datahandler could not be setup.');
             return false;
         }
-        if (! $this->shouldProcessTable($table)) {
+        if (! $this->dataHandler->supportsTable($table)) {
             $this->logger->debug('Table is not allowed.', [$table]);
             return false;
         }
@@ -148,23 +148,12 @@ class DataHandler implements Singleton
         return true;
     }
 
-    /**
-     * @param string $table
-     * @return bool
-     */
-    protected function shouldProcessTable($table)
-    {
-        return in_array($table, $this->dataHandler->getAllowedTablesForIndexing());
-    }
-
     /**
      * Wrapper to allow unit testing.
      *
-     * @param string $table
+     * @return array|null
-     * @param int $uid
-     * @return null|array<String>
      */
-    protected function getRecord($table, $uid)
+    protected function getRecord(string $table, int $uid)
     {
         return BackendUtility::getRecord($table, $uid);
     }

Classes/Integration/Form/Finisher/DataHandlerFinisher.php

@@ -0,0 +1,69 @@
+<?php
+namespace Codappix\SearchCore\Integration\Form\Finisher;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use TYPO3\CMS\Form\Domain\Finishers\AbstractFinisher;
+use TYPO3\CMS\Form\Domain\Finishers\Exception\FinisherException;
+
+/**
+ * Integrates search_core indexing into TYPO3 Form extension.
+ *
+ * Add this finisher AFTER all database operations, as search_core will fetch
+ * information from database.
+ */
+class DataHandlerFinisher extends AbstractFinisher
+{
+    /**
+     * @var \Codappix\SearchCore\Domain\Service\DataHandler
+     * @inject
+     */
+    protected $dataHandler;
+
+    /**
+     * @var array
+     */
+    protected $defaultOptions = [
+        'indexIdentifier' => null,
+        'recordUid' => null,
+        'action' => '',
+    ];
+
+    protected function executeInternal()
+    {
+        $action = $this->parseOption('action');
+        $record = ['uid' => (int) $this->parseOption('recordUid')];
+        $tableName = $this->parseOption('indexIdentifier');
+
+        if ($action === '' || $tableName === '' || !is_string($tableName) || $record['uid'] === 0) {
+            throw new FinisherException('Not all necessary options were set.', 1510313095);
+        }
+
+        switch ($action) {
+            case 'update':
+            case 'add':
+                $this->dataHandler->update($tableName, $record);
+                break;
+            case 'delete':
+                $this->dataHandler->delete($tableName, (string) $record['uid']);
+                break;
+        }
+    }
+}

Classes/Utility/FrontendUtility.php

@@ -0,0 +1,36 @@
+<?php
+namespace Codappix\SearchCore\Utility;
+
+/*
+ * Copyright (C) 2018 Justus Moroni <developer@leonmrni.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use TYPO3\CMS\Backend\Utility\BackendUtility;
+use TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController;
+
+/**
+ * Overwrite BackendUtility to use in frontend.
+ * LanguageService was only usable in backend.
+ */
+class FrontendUtility extends BackendUtility
+{
+    protected static function getLanguageService() : TypoScriptFrontendController
+    {
+        return $GLOBALS['TSFE'];
+    }
+}

Configuration/TCA/Overrides/tt_content.php

@@ -0,0 +1,3 @@
+<?php
+
+$GLOBALS['TCA']['tt_content']['types']['list']['subtypes_excludelist']['searchcore_search'] = 'recursive,pages';

Configuration/TypoScript/constants.txt

@@ -8,20 +8,14 @@ plugin {
                 }
             }
 
-            indexer {
+            indexing {
-                tca {
+                tt_content {
-                    # Pages are not supported yet, see
+                    additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu', 'shortcut', 'search', 'login') AND tt_content.bodytext != ''
-                    # https://github.com/DanielSiepmann/search_core/issues/24 but
+                }
-                    # should also be added, together with additionalWhereClause
-                    # based on doktypes
-                    allowedTables = tt_content
 
-                    tt_content {
+                pages {
-                        additionalWhereClause (
+                    additionalWhereClause = pages.doktype NOT IN (3, 199, 6, 254, 255)
-                            pages.doktype NOT IN (3, 199)
+                    abstractFields = abstract, description, bodytext
-                            AND tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu', 'shortcut', 'search', 'login')
-                        )
-                    }
                 }
             }
         }

Configuration/TypoScript/setup.txt

@@ -8,13 +8,23 @@ plugin {
                 }
             }
 
-            indexer {
+            indexing {
-                tca {
+                # Not for direct indexing therefore no indexer.
-                    allowedTables = {$plugin.tx_searchcore.settings.indexer.tca.allowedTables}
+                # Used to configure tt_content fetching while indexing pages
+                tt_content {
+                    additionalWhereClause = {$plugin.tx_searchcore.settings.indexing.tt_content.additionalWhereClause}
+                }
 
-                    tt_content {
+                pages {
-                        additionalWhereClause = {$plugin.tx_searchcore.settings.indexer.tca.tt_content.additionalWhereClause}
+                    indexer = Codappix\SearchCore\Domain\Index\TcaIndexer\PagesIndexer
-                    }
+                    additionalWhereClause = {$plugin.tx_searchcore.settings.indexing.pages.additionalWhereClause}
+                    abstractFields = {$plugin.tx_searchcore.settings.indexing.pages.abstractFields}
+                }
+            }
+
+            searching {
+                fields {
+                    query = _all
                 }
             }
         }

Documentation/source/changelog.rst

@@ -0,0 +1,11 @@
+Changelog
+=========
+
+.. toctree::
+   :maxdepth: 1
+   :glob:
+
+   changelog/20180409-25-provide-sys-language-uid
+   changelog/20180408-131-respect-page-cache-clear
+   changelog/20180408-introduce-php70-type-hints
+   changelog/20180406-120-facet-configuration

Documentation/source/changelog/20180406-120-facet-configuration.rst

@@ -0,0 +1,40 @@
+Breacking Change 120 "Pass facets configuration to Elasticsearch"
+=================================================================
+
+In order to allow arbitrary facet configuration, we do not process the facet configuration anymore.
+Instead integrators are able to configure facets for search service "as is". We just pipe the
+configuration through.
+
+Therefore the following, which worked before, does not work anymore:
+
+.. code-block:: typoscript
+   :linenos:
+   :emphasize-lines: 4
+
+    plugin.tx_searchcore.settings.search {
+        facets {
+            category {
+                field = categories
+            }
+        }
+    }
+
+Instead you have to provide the full configuration yourself:
+
+.. code-block:: typoscript
+   :linenos:
+   :emphasize-lines: 4,6
+
+    plugin.tx_searchcore.settings.search {
+        facets {
+            category {
+                terms {
+                    field = categories
+                }
+            }
+        }
+    }
+
+You need to add line 4 and 6, the additional level ``terms`` for Elasticsearch.
+
+See :issue:`120`.

Documentation/source/changelog/20180408-131-respect-page-cache-clear.rst

@@ -0,0 +1,14 @@
+Feature 131 "Pages do not get indexed if content has changed"
+=============================================================
+
+Previously we only used DataHandler hooks triggered when processing records. This way we did not
+index a page when content has changed.
+
+We now also use cache clear hooks of DataHandler to index pages whenever their cache get cleared.
+This way we also index a page if an integrator configured to clear further pages if content was
+changed.
+
+Still there are limitations. We do not get informed for pages which got cleared due to attached
+caches via TypoScript.
+
+See :issue:`131`.

Documentation/source/changelog/20180408-introduce-php70-type-hints.rst

@@ -0,0 +1,12 @@
+Breacking Change "Introduce PHP 7.0 TypeHints"
+==============================================
+
+As PHP evolved, we now migrate the whole code base to use PHP 7.0 type hints.
+We do not use PHP 7.1 Type Hints, as some customers still need PHP 7.0 support.
+
+Also we added missing methods to interfaces, that were already used in code.
+
+As this leads to changed method signatures, most custom implementations of interfaces, or overwrites
+of existing methods are broken.
+
+To fix, just update the signatures as pointed out by PHP while running the code.

Documentation/source/changelog/20180409-25-provide-sys-language-uid.rst

@@ -0,0 +1,16 @@
+FEATURE 25 "Respect multiple languages" - Provide sys_language_uid
+==================================================================
+
+Previously we did not fetch ``sys_language_uid`` field from database. This prevented everyone from
+working with multiple languages.
+By not removing the field it gets indexed and provides a very basic way of implementing multiple
+languages.
+At least it's now possible to filter search results by current language for now. Still the records
+are not "valid" as we do not add overlays for now.
+
+This is a first step into full multi language support.
+
+Martin Hummer already has a basic proof of concept, based on :ref:`concepts_dataprocessing` working,
+depending on ``sys_language_uid``.
+
+See :issue:`25`.

Documentation/source/concepts.rst

@@ -3,28 +3,47 @@
 Concepts
 ========
 
-The extension is built with the following concepts in mind.
+The main concept is to provide a foundation where other developers can profit from, to provide
+integrations into search services like Elasticsearch, Algolia, ... .
+
+Our code contains the following concepts which should be understand:
 
 .. _concepts_connections:
 
 Connections
 -----------
 
-It should be possible to use different search services like elasticsearch and solr out of the box.
+Different search services can provide integrations. ``search_core`` only provides abstractions and
-If a service is not contained, it should be possible to implement the necessary part by implementing
+interfaces. The main purpose is to provide a stable API between TYPO3 and concrete connection.
-the necessary interfaces and configuring the extension to use the new connection.
 
-Also it should be possible to use multiple connections at once. This way multiple search services
+For information about implementing a new connection, take a look at :ref:`development_connection`.
-can be used in the same installation.
-
-Currently only :ref:`Elasticsearch` is provided.
 
 .. _concepts_indexing:
 
 Indexing
 --------
 
-The indexing is done by one of the available indexer. It should be possible to define the indexer to
+Indexing is the process of collecting and preparing data, before sending it to a Connection.
-use for certein document types. Also it should be possible to write custom indexer to use.
+The indexing is done by one of the available indexer. Indexer are identified by a key, as configured
+in TypoScript.
 
-Currently only the :ref:`TcaIndexer` is provided.
+Currently :ref:`TcaIndexer` and :ref:`PagesIndexer` are provided.
+
+For information about implementing a new indexer, take a look at :ref:`development_indexer`.
+
+.. _concepts_dataprocessing:
+
+DataProcessing
+^^^^^^^^^^^^^^
+
+Before data is transfered to search service, it can be processed by "DataProcessors" like already
+known by :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing` of :ref:`t3tsref:cobj-fluidtemplate`.
+The same is true for retrieved search results. They can be processed again by "DataProcessors" to
+prepare data for display in Templates or further usage.
+
+This should keep indexers simple and move logic to DataProcessors. This makes most parts highly
+flexible as integrators are able to configure DataProcessors and change their order.
+
+Configuration is done through TypoScript, see :ref:`dataprocessors`.
+
+For information about implementing a new DataProcessor, take a look at :ref:`development_dataprocessor`.

Documentation/source/conf.py

@@ -51,7 +51,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'TYPO3 Extension search_core'
-copyright = u'2016, Daniel Siepmann'
+copyright = u'2016 - 2018, Daniel Siepmann'
 author = u'Daniel Siepmann'
 
 # The version info for the project you're documenting, acts as replacement for
@@ -59,9 +59,9 @@ author = u'Daniel Siepmann'
 # built documents.
 #
 # The short X.Y version.
-version = u'1.0.0'
+version = u'0.0.1'
 # The full version, including alpha/beta/rc tags.
-release = u'1.0.0'
+release = u'0.0.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -133,7 +133,7 @@ html_theme_options = {
 
 # The name for this set of Sphinx documents.
 # "<project> v<release> documentation" by default.
-#html_title = u'TYPO3 Extension search_core v1.0.0'
+#html_title = u'TYPO3 Extension search_core v0.0.1'
 
 # A shorter title for the navigation bar.  Default is the same as html_title.
 #html_short_title = None
@@ -150,7 +150,7 @@ html_theme_options = {
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+# html_static_path = ['_static']
 
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
@@ -304,6 +304,7 @@ texinfo_documents = [
 # Example configuration for intersphinx: refer to the Python standard library.
 intersphinx_mapping = {
     't3tcaref': ('https://docs.typo3.org/typo3cms/TCAReference/', None),
+    't3tsref': ('https://docs.typo3.org/typo3cms/TyposcriptReference/', None),
 }
 extlinks = {
     'project': ('https://github.com/Codappix/search_core/projects/%s', 'Github project: '),

Documentation/source/configuration.rst

@@ -5,9 +5,12 @@
 Configuration
 =============
 
-The extension offers the following configuration options through TypoScript. If you overwrite them
+Installation wide configuration is handled inside of the extension manager. Just check out the
-through `setup` make sure to keep them in the `module` area as they will be accessed from backend
+options there, they all have labels.
-mode of TYPO3. Do so by placing the following line at the end::
+
+Everything else is configured through TypoScript. If you overwrite them through `setup` make sure to
+keep them in the `module` area as they will be accessed from backend mode of TYPO3 for indexing. Do
+so by placing the following line at the end::
 
     module.tx_searchcore < plugin.tx_searchcore
 
@@ -23,12 +26,10 @@ Here is the example default configuration that's provided through static include
 
 .. literalinclude:: ../../Configuration/TypoScript/constants.txt
    :language: typoscript
-   :linenos:
    :caption: Static TypoScript Constants
 
 .. literalinclude:: ../../Configuration/TypoScript/setup.txt
    :language: typoscript
-   :linenos:
    :caption: Static TypoScript Setup
 
 .. _configuration_options:
@@ -36,143 +37,14 @@ Here is the example default configuration that's provided through static include
 Options
 -------
 
-The following section contains the different options, e.g. for :ref:`connections` and
+The following sections contains the different options grouped by their applied area, e.g. for
-:ref:`indexer`: ``plugin.tx_searchcore.settings.connection`` or
+:ref:`connections` and :ref:`indexer`: ``plugin.tx_searchcore.settings.connection`` or
-``plugin.tx_searchcore.settings.index``.
+``plugin.tx_searchcore.settings.indexing``:
 
-.. _configuration_options_connection:
+.. toctree::
+   :maxdepth: 1
+   :glob:
 
-connections
+   configuration/connections
-^^^^^^^^^^^
+   configuration/indexing
-
+   configuration/searching
-Holds settings regarding the different possible connections for search services like Elasticsearch
-or Solr.
-
-Configured as::
-
-    plugin {
-        tx_searchcore {
-            settings {
-                connections {
-                    connectionName {
-                        // the settings
-                    }
-                }
-            }
-        }
-    }
-
-Where ``connectionName`` is one of the available :ref:`connections`.
-
-The following settings are available. For each setting its documented which connection consumes it.
-
-.. _host:
-
-``host``
-""""""""
-
-    Used by: :ref:`Elasticsearch`.
-
-    The host, e.g. ``localhost`` or an IP where the search service is reachable from TYPO3
-    installation.
-
-    Example::
-
-        plugin.tx_searchcore.settings.connections.elasticsearch.host = localhost
-
-.. _port:
-
-``port``
-""""""""
-
-    Used by: :ref:`Elasticsearch`.
-
-    The port where search service is reachable. E.g. default ``9200`` for Elasticsearch.
-
-    Example::
-
-        plugin.tx_searchcore.settings.connections.elasticsearch.port = 9200
-
-
-.. _configuration_options_index:
-
-index
-^^^^^
-
-Holds settings regarding the indexing, e.g. of TYPO3 records, to search services.
-
-Configured as::
-
-    plugin {
-        tx_searchcore {
-            settings {
-                indexer {
-                    indexerName {
-                        // the settings
-                    }
-                }
-            }
-        }
-    }
-
-Where ``indexerName`` is one of the available :ref:`indexer`.
-
-The following settings are available. For each setting its documented which indexer consumes it.
-
-.. _allowedTables:
-
-``allowedTables``
-"""""""""""""""""
-
-    Used by: :ref:`TcaIndexer`.
-
-    Defines which TYPO3 tables are allowed to be indexed. Only white listed tables will be processed
-    through Command Line Interface and Hooks.
-
-    Contains a comma separated list of table names. Spaces are trimmed.
-
-    Example::
-
-        plugin.tx_searchcore.settings.indexer.tca.allowedTables = tt_content, fe_users
-
-.. _rootLineBlacklist:
-
-``rootLineBlacklist``
-"""""""""""""""""""""
-
-    Used by: :ref:`TcaIndexer`.
-
-    Defines a blacklist of page uids. Records below any of these pages, or subpages, are not
-    indexed. This allows you to define areas that should not be indexed.
-    The page attribute *No Search* is also taken into account to prevent indexing records from only one
-    page without recursion.
-
-    Contains a comma separated list of page uids. Spaces are trimmed.
-
-    Example::
-
-        plugin.tx_searchcore.settings.index.tca.rootLineBlacklist = 3, 10, 100
-
-Also it's possible to define some behaviour for the different document types. In context of TYPO3
-tables are used as document types 1:1. It's possible to configure different tables. The following
-options are available:
-
-.. _additionalWhereClause:
-
-``additionalWhereClause``
-"""""""""""""""""""""""""
-
-    Used by: :ref:`TcaIndexer`.
-
-    Add additional SQL to where clauses to determine indexable records from the table. This way you
-    can exclude specific records like ``tt_content`` records with specific ``CType`` values or
-    something else. E.g. you can add a new field to the table to exclude records from indexing.
-
-    Example::
-
-        plugin.tx_searchcore.settings.index.tca.tt_content.additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu')
-
-    .. attention::
-
-        Make sure to prefix all fields with the corresponding table name. The selection from
-        database will contain joins and can lead to SQL errors if a field exists in multiple tables.

Documentation/source/configuration/connections.rst

@@ -0,0 +1,48 @@
+.. _configuration_options_connection:
+
+Connections
+===========
+
+Holds settings regarding the different possible connections for search services like Elasticsearch
+or Algolia.
+
+Configured as::
+
+    plugin {
+        tx_searchcore {
+            settings {
+                connections {
+                    connectionName {
+                        // the settings
+                    }
+                }
+            }
+        }
+    }
+
+Where ``connectionName`` is one of the available :ref:`connections`.
+
+The following settings are available.
+
+.. _host:
+
+``host``
+--------
+
+The host, e.g. ``localhost`` or an IP where the search service is reachable from TYPO3
+installation.
+
+Example::
+
+    plugin.tx_searchcore.settings.connections.elasticsearch.host = localhost
+
+.. _port:
+
+``port``
+--------
+
+The port where search service is reachable. E.g. default ``9200`` for Elasticsearch.
+
+Example::
+
+    plugin.tx_searchcore.settings.connections.elasticsearch.port = 9200

Documentation/source/configuration/dataProcessing/ContentObjectDataProcessorAdapterProcessor.rst

@@ -0,0 +1,32 @@
+``Codappix\SearchCore\DataProcessing\ContentObjectDataProcessorAdapterProcessor``
+=================================================================================
+
+Will execute an existing TYPO3 data processor.
+
+Possible Options:
+
+``_dataProcessor``
+    Necessary, defined which data processor to apply. Provide the same as you would to call the
+    processor.
+``_table``
+    Defines the "current" table as used by some processors, e.g.
+    ``TYPO3\CMS\Frontend\DataProcessing\FilesProcessor``.
+
+All further options are passed to the configured data processor. Therefore they are documented at
+each data processor.
+
+Example::
+
+    plugin.tx_searchcore.settings.searching.dataProcessing {
+        1 = Codappix\SearchCore\DataProcessing\ContentObjectDataProcessorAdapterProcessor
+        1 {
+            _table = pages
+            _dataProcessor = TYPO3\CMS\Frontend\DataProcessing\FilesProcessor
+
+            references.fieldName = media
+            as = images
+        }
+    }
+
+The above example will create a new field ``images`` with resolved FAL relations from ``media``
+field.

Documentation/source/configuration/dataProcessing/CopyToProcessor.rst

@@ -0,0 +1,23 @@
+``Codappix\SearchCore\DataProcessing\CopyToProcessor``
+======================================================
+
+Will copy contents of fields to other fields.
+
+Possible Options:
+
+``to``
+    Defines the field to copy the values into. All values not false will be copied at the moment.
+
+Example::
+
+    plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing {
+        1 = Codappix\SearchCore\DataProcessing\CopyToProcessor
+        1 {
+            to = all
+        }
+        2 = Codappix\SearchCore\DataProcessing\CopyToProcessor
+        2 {
+            to = spellcheck
+        }
+    }
+

Documentation/source/configuration/dataProcessing/GeoPointProcessor.rst

@@ -0,0 +1,27 @@
+``Codappix\SearchCore\DataProcessing\GeoPointProcessor``
+========================================================
+
+Will create a new field, ready to use as GeoPoint field for Elasticsearch.
+
+Possible Options:
+
+``to``
+    Defines the field to create as GeoPoint.
+``lat``
+    Defines the field containing the latitude.
+``lon``
+    Defines the field containing the longitude.
+
+Example::
+
+    plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing {
+        1 = Codappix\SearchCore\DataProcessing\GeoPointProcessor
+        1 {
+            to = location
+            lat = lat
+            lon = lng
+        }
+    }
+
+The above example will create a new field ``location`` as GeoPoint with latitude fetched from field
+``lat`` and longitude fetched from field ``lng``.

Documentation/source/configuration/dataProcessing/RemoveProcessor.rst

@@ -0,0 +1,23 @@
+``Codappix\SearchCore\DataProcessing\RemoveProcessor``
+======================================================
+
+Will remove fields from record.
+
+Possible Options:
+
+``fields``
+    Comma separated list of fields to remove from record.
+
+Example::
+
+    plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing {
+        1 = Codappix\SearchCore\DataProcessing\RemoveProcessor
+        1 {
+            fields = description
+        }
+        2 = Codappix\SearchCore\DataProcessing\RemoveProcessor
+        2 {
+            fields = description, another_field
+        }
+    }
+

Documentation/source/configuration/indexing.rst

@@ -0,0 +1,152 @@
+.. _configuration_options_index:
+
+Indexing
+========
+
+Holds settings regarding the indexing, e.g. of TYPO3 records, to search services.
+
+Configured as::
+
+    plugin {
+        tx_searchcore {
+            settings {
+                indexing {
+                    identifier {
+                        indexer = FullyQualifiedClassname
+                        // the settings
+                    }
+                }
+            }
+        }
+    }
+
+Where ``identifier`` is up to you, but should match table names to make :ref:`TcaIndexer` work.
+
+The following settings are available. For each setting its documented which indexer consumes it.
+
+.. _rootLineBlacklist:
+
+rootLineBlacklist
+-----------------
+
+Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`.
+
+Defines a blacklist of page uids. Records below any of these pages, or subpages, are not
+indexed. This allows you to define areas that should not be indexed.
+The page attribute *No Search* is also taken into account to prevent indexing records from only one
+page without recursion.
+
+Contains a comma separated list of page uids. Spaces are trimmed.
+
+Example::
+
+    plugin.tx_searchcore.settings.indexing.pages.rootLineBlacklist = 3, 10, 100
+
+.. _additionalWhereClause:
+
+additionalWhereClause
+---------------------
+
+Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`.
+
+Add additional SQL to where clauses to determine indexable records from the table. This way you
+can exclude specific records like ``tt_content`` records with specific ``CType`` values or
+something else.
+
+Example::
+
+    plugin.tx_searchcore.settings.indexing.tt_content.additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu')
+
+.. attention::
+
+    Make sure to prefix all fields with the corresponding table name. The selection from
+    database might contain joins and can lead to SQL errors if a field exists in multiple tables.
+
+.. _abstractFields:
+
+abstractFields
+--------------
+
+Used by: :ref:`PagesIndexer`.
+
+.. note::
+
+   Will be migrated to :ref:`dataprocessors` in the future.
+
+Define which field should be used to provide the auto generated field "search_abstract".
+The fields have to exist in the record to be indexed. Therefore fields like ``content`` are also
+possible.
+
+Example::
+
+    # As last fallback we use the content of the page
+    plugin.tx_searchcore.settings.indexing.pages.abstractFields := addToList(content)
+
+Default::
+
+    abstract, description, bodytext
+
+.. _mapping:
+
+mapping
+-------
+
+Used by: :ref:`connection_elasticsearch` connection while indexing.
+
+Define mapping for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/mapping.html
+You are able to define the mapping for each property / column.
+
+Example::
+
+    plugin.tx_searchcore.settings.indexing.tt_content.mapping {
+        CType {
+            type = keyword
+        }
+    }
+
+The above example will define the ``CType`` field of ``tt_content`` as ``type: keyword``. This
+makes building a facet possible.
+
+.. _index:
+
+index
+-----
+
+Used by: :ref:`connection_elasticsearch` connection while indexing.
+
+Define index for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/indices-create-index.html
+
+Example::
+
+    plugin.tx_searchcore.settings.indexing.tt_content.index {
+        analysis {
+            analyzer {
+                ngram4 {
+                    type = custom
+                    tokenizer = ngram4
+                    char_filter = html_strip
+                    filter = lowercase, asciifolding
+                }
+            }
+
+            tokenizer {
+                ngram4 {
+                    type = ngram
+                    min_gram = 4
+                    max_gram = 4
+                }
+            }
+        }
+    }
+
+``char_filter`` and ``filter`` are a comma separated list of options.
+
+.. _indexing_dataProcessing:
+
+dataProcessing
+--------------
+
+Used by: All connections while indexing, due to implementation inside ``AbstractIndexer``.
+
+Configure modifications on each document before sending it to the configured connection.
+For full documentation check out :ref:`dataprocessors`.

Documentation/source/configuration/searching.rst

@@ -0,0 +1,261 @@
+.. _configuration_options_search:
+
+Searching
+=========
+
+.. _size:
+
+size
+----
+
+Defined how many search results should be fetched to be available in search result.
+
+Example::
+
+    plugin.tx_searchcore.settings.searching.size = 50
+
+Default is ``10``.
+
+.. _facets:
+
+facets
+------
+
+Define aggregations for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-aggregations-bucket-terms-aggregation.html
+
+Example::
+
+    category {
+        terms {
+            field = categories
+        }
+    }
+
+    month {
+        date_histogram {
+            field = released
+            interval = month
+            format = Y-MM-01
+            order {
+                _time = desc
+            }
+        }
+    }
+
+
+The above example will provide a facet with options for all found ``categories`` results together
+with a count. Also a facet for ``released`` will be provided.
+
+.. _filter:
+
+filter
+------
+
+Define filter that should be set for all search requests.
+
+Example::
+
+    plugin.tx_searchcore.settings.searching.filter {
+        property = value
+    }
+
+Also see :ref:`mapping.filter` to map incoming request information, e.g. from a ``select``, to build
+more complex filters.
+
+For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``keyword``.
+
+.. _minimumShouldMatch:
+
+minimumShouldMatch
+------------------
+
+Define the minimum match for Elasticsearch, have a look at the official docs:
+https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-minimum-should-match.html
+
+Example::
+
+    plugin.tx_searchcore.settings.searching.minimumShouldMatch = 50%
+
+.. _boost:
+
+boost
+-----
+
+Define fields that should boost the score for results.
+
+Example::
+
+    plugin.tx_searchcore.settings.searching.boost {
+        search_title = 3
+        search_abstract = 1.5
+    }
+
+For further information take a look at
+https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_boosting_query_clauses.html
+
+.. _fieldValueFactor:
+
+fieldValueFactor
+----------------
+
+Define a field to use as a factor for scoring. The configuration is passed through to Elasticsearch
+``field_value_factor``, see:
+https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor
+
+Example::
+
+    plugin.tx_searchcore.settings.searching.field_value_factor {
+        field = rootlineLevel
+        modifier = reciprocal
+        factor = 2
+        missing = 1
+    }
+
+.. _mapping.filter:
+
+mapping.filter
+--------------
+
+Allows to configure filter more in depth. If a filter with the given key exists, the TypoScript will
+be added.
+
+E.g. you submit a filter in form of:
+
+.. code-block:: html
+
+   <f:comment>
+       Due to TYPO3 7.x fluid limitations, we build this input ourself.
+       No longer necessary in 8 and above
+   </f:comment>
+   <select name="tx_searchcore_search[searchRequest][filter][month][from]" class="_control" >
+       <option value="">Month</option>
+       <f:for each="{searchResult.facets.month.options}" as="month">
+           <f:if condition="{month.count}">
+               <option
+                   value="{month.displayName -> f:format.date(format: 'Y-m')}"
+                   {f:if(condition: '{searchRequest.filter.month.from} == {month.displayName -> f:format.date(format: \'Y-m\')}', then: 'selected="true"')}
+               >{month.displayName -> f:format.date(format: '%B %Y')}</option>
+           </f:if>
+       </f:for>
+   </select>
+   <select name="tx_searchcore_search[searchRequest][filter][month][to]" class="_control" >
+       <option value="">Month</option>
+       <f:for each="{searchResult.facets.month.options}" as="month">
+           <f:if condition="{month.count}">
+               <option
+                   value="{month.displayName -> f:format.date(format: 'Y-m')}"
+                   {f:if(condition: '{searchRequest.filter.month.from} == {month.displayName -> f:format.date(format: \'Y-m\')}', then: 'selected="true"')}
+               >{month.displayName -> f:format.date(format: '%B %Y')}</option>
+           </f:if>
+       </f:for>
+   </select>
+
+This will create a ``month`` filter with sub properties. To make this filter actually work, you
+can add the following TypoScript, which will be added to the filter::
+
+    mapping {
+        filter {
+            month {
+                type = range
+                field = released
+                raw {
+                    format = yyyy-MM
+                }
+
+                fields {
+                    gte = from
+                    lte = to
+                }
+            }
+        }
+    }
+
+``fields`` has a special meaning here. This will actually map the properties of the filter to fields
+in Elasticsearch. On the left hand side is the Elasticsearch field name, on the right side the one
+submitted as a filter.
+
+The ``field``, in above example ``released``, will be used as the Elasticsearch field for
+filtering. This way you can use arbitrary filter names and map them to existing Elasticsearch fields.
+
+Everything that is configured inside ``raw`` is passed, as is, to search service, e.g.
+Elasticsearch.
+
+.. _fields:
+
+fields
+------
+
+Defines the fields to fetch and search from Elasticsearch. With the following sub keys:
+
+``query`` defines the fields to search in. Configure a comma separated list of fields to search in.
+This is necessary if you have configured special mapping for some fields, or just want to search
+some fields. The following is an example configuration::
+
+    fields {
+        query = _all, city
+    }
+
+The following sub properties configure the fields to fetch from Elasticsearch:
+
+First ``stored_fields`` which is a list of comma separated fields which actually exist and will be
+added. Typically you will use ``_source`` to fetch the whole indexed fields.
+
+Second is ``script_fields``, which allow you to configure scripted fields for Elasticsearch.
+An example might look like the following::
+
+    fields {
+        script_fields {
+            distance {
+                condition = {request.filter.distance.location}
+                script {
+                    params {
+                        lat = {request.filter.distance.location.lat -> f:format.number()}
+                        lon = {request.filter.distance.location.lon -> f:format.number()}
+                    }
+                    lang = painless
+                    inline = doc["location"].arcDistance(params.lat,params.lon) * 0.001
+                }
+            }
+        }
+    }
+
+In above example we add a single ``script_field`` called ``distance``. We add a condition when this
+field should be added. The condition will be parsed as Fluidtemplate and is casted to bool via PHP.
+If the condition is true, or no ``condition`` exists, the ``script_field`` will be added to the
+query. The ``condition`` will be removed and everything else is submitted one to one to
+Elasticsearch, except each property is run through Fluidtemplate, to allow you to use information
+from search request, e.g. to insert latitude and longitude from a filter, like in the above example.
+
+.. _sort:
+
+sort
+----
+
+Sort is handled like :ref:`fields`.
+
+.. _mode:
+
+mode
+----
+
+Used by: Controller while preparing action.
+
+Define to switch from search to filter mode.
+
+Example::
+
+    plugin.tx_searchcore.settings.searching {
+        mode = filter
+    }
+
+Only ``filter`` is allowed as value, as ``search`` is default behaviour. Using ``filter`` will
+trigger a search to provide data while visiting the page, possible :ref:`filter` allow you to build
+pages like "News".
+
+.. _searching_dataprocessing:
+
+dataProcessing
+--------------
+
+Configure modifications on each document before returning search result.
+For full documentation check out :ref:`dataprocessors`.

Documentation/source/connections.rst

@@ -5,22 +5,25 @@ Connections
 
 See Concept of :ref:`concepts_connections` for further background information.
 
-The extension provides the following connections out of the box:
+For information about implementing a new connection, take a look at :ref:`development_connection`.
 
-.. _Elasticsearch:
+The following connections were developed, or are in development, for ``search_core``:
+
+.. _connection_elasticsearch:
 
 Elasticsearch
 -------------
 
-Integrates `elastic Elasticsearch`_ using `elastica`_ into TYPO3.
+Support for `Elasticsearch`_ is provided out of the box by `search_core` at the moment.
 
-Provides basic support like indexing without mappings and full text search at the moment.
+.. _Elasticsearch: https://www.elastic.co/products/elasticsearch
 
-The connection is configurable through the following options:
+.. _connection_algolia:
 
-* :ref:`host`
+Algolia
+-------
 
-* :ref:`port`
+`search_algolia`_ will integrate `Algolia`_ and is currently under development by Martin Hummer.
 
-.. _elastic Elasticsearch: https://www.elastic.co/products/elasticsearch
+.. _search_algolia: https://github.com/martinhummer/search_algolia/
-.. _elastica: http://elastica.io/
+.. _Algolia: https://www.algolia.com/

Documentation/source/dataprocessors.rst

@@ -0,0 +1,98 @@
+.. _dataprocessors:
+
+DataProcessors
+==============
+
+See Concept of :ref:`concepts_dataprocessing` for further background information.
+
+For information about implementing a new DataProcessor, take a look at
+:ref:`development_dataprocessor`.
+
+Same as provided by TYPO3 for :ref:`t3tsref:cobj-fluidtemplate` through
+:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`.
+
+.. _dataprocessors_usage:
+
+Usage
+-----
+
+All processors are applied in configured order. Allowing to work with already processed data.
+They can be applied during indexing and for search results.
+
+Example for indexing::
+
+    plugin.tx_searchcore.settings.indexing.pages.dataProcessing {
+        1 = Codappix\SearchCore\DataProcessing\CopyToProcessor
+        1 {
+            to = search_spellcheck
+        }
+
+        2 = Codappix\SearchCore\DataProcessing\CopyToProcessor
+        2 {
+            to = search_all
+        }
+    }
+
+The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards
+all fields, including ``search_spellcheck`` will be copied to ``search_all``.
+
+Example for search results::
+
+    plugin.tx_searchcore.settings.searching.dataProcessing {
+        1 = Codappix\SearchCore\DataProcessing\CopyToProcessor
+        1 {
+            to = search_spellcheck
+        }
+
+        2 = Codappix\SearchCore\DataProcessing\CopyToProcessor
+        2 {
+            to = search_all
+        }
+    }
+
+The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards
+all fields, including ``search_spellcheck`` will be copied to ``search_all``.
+
+.. _dataprocessors_availableDataProcessors:
+
+Available DataProcessors
+------------------------
+
+.. toctree::
+    :maxdepth: 1
+    :glob:
+
+    /configuration/dataProcessing/ContentObjectDataProcessorAdapterProcessor
+    /configuration/dataProcessing/CopyToProcessor
+    /configuration/dataProcessing/GeoPointProcessor
+    /configuration/dataProcessing/RemoveProcessor
+
+.. _dataprocessors_plannedDataProcessors:
+
+Planned DataProcessors
+----------------------
+
+    ``Codappix\SearchCore\DataProcessing\ReplaceProcessor``
+        Will execute a search and replace on configured fields.
+
+    ``Codappix\SearchCore\DataProcessing\RootLevelProcessor``
+        Will attach the root level to the record.
+
+    ``Codappix\SearchCore\DataProcessing\ChannelProcessor``
+        Will add a configurable channel to the record, e.g. if you have different areas in your
+        website like "products" and "infos".
+
+    ``Codappix\SearchCore\DataProcessing\RelationResolverProcessor``
+        Resolves all relations using the TCA.
+        This is currently done through indexer.
+
+.. Of course you are able to provide further processors. Just implement
+.. ``Codappix\SearchCore\DataProcessing\ProcessorInterface`` and use the FQCN (=Fully qualified
+.. class name) as done in the examples above.
+
+.. By implementing also the same interface as necessary for TYPO3
+.. :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`, you are able to reuse the same code
+.. also for Fluid to prepare the same record fetched from DB for your fluid.
+
+.. Dependency injection is possible inside of processors, as we instantiate through extbase
+.. ``ObjectManager``.

Documentation/source/development.rst

@@ -1,70 +1,16 @@
 .. highlight:: bash
 
-.. _contribution:
-
-Contribution
-============
-
-Everyone is welcome to contribute, whether it's code, issues, feature requests or any other kind.
-
-Below is a documentation what to respect during contributions.
-
-.. _contribution_setup:
-
-Setup
------
-
-To start contributions regarding code, make sure your environment matches the following
-requirements:
-
-* composer is executable
-
-* PHP on CLI is executable
-
-* MySQL is up and running with user *dev* and password *dev* on *127.0.0.1* or to overwrite the
-  environment variables, see :file:`Makefile`.
-  And MySQL is not set to strict mode as TYPO3 doesn't support strict mode, see
-  https://review.typo3.org/#/c/26725/3/INSTALL.md.
-
-* Elasticsearch is installed and up and running on *localhost:9200*.
-
-Then setup your system::
-
-    git clone git@github.com:DanielSiepmann/search_core.git \
-        && cd search_core \
-        && export typo3DatabaseName="searchcoretest62" \
-        && export TYPO3_VERSION="~6.2" \
-        && make install \
-        && make functionalTests
-
-If all tests are okay, start your work.
-
-If you are working with multiple TYPO3 versions make sure to export `typo3DatabaseName` and
-`TYPO3_VERSION` in your environment like::
-
-    export typo3DatabaseName="searchcoretest76"
-    export TYPO3_VERSION="~7.6"
-
-Also run the install command for each version before running any tests. Only this will make sure you
-are testing against the actual TYPO3 Version and database scheme.
-
-.. _contribution_development:
-
 Development
------------
+===========
 
-All changes are introduced through pull requests at `Github`_ and should contain the following:
+There are some ways we will cover here. One is how you can develop own parts like Indexer,
+DataProcessor and Connection. The other is how to contribute.
 
-* Adjusted tests if tests existed before. Otherwise they will break on `travis-ci`_.
+.. toctree::
+   :maxdepth: 1
+   :glob:
 
-* New tests whenever possible and useful.
+   development/indexer
-
+   development/dataProcessor
-* Code has to follow `PSR-2`_.
+   development/connection
-
+   development/contribution
-* Adjusted documentation.
-
-* Make sure to follow the documented :ref:`concepts`.
-
-.. _Github: https://github.com/DanielSiepmann/search_core
-.. _travis-ci: https://travis-ci.org/
-.. _PSR-2: http://www.php-fig.org/psr/psr-2/

Documentation/source/development/connection.rst

@@ -0,0 +1,11 @@
+.. _development_connection:
+
+Develop a new Connection
+========================
+
+Make sure you understood :ref:`concepts_connections`.
+
+Each Connection has to be a single class which implements
+``Codappix\SearchCore\Connection\ConnectionInterface``.
+
+Dependency Injection is working for custom Connections.

Documentation/source/development/contribution.rst

@@ -0,0 +1,69 @@
+.. _contribution:
+
+Contribution
+============
+
+Everyone is welcome to contribute, whether it's code, issues, feature requests or any other kind.
+
+Below is a documentation what to respect during contributions.
+
+.. _contribution_setup:
+
+Setup
+-----
+
+To start contributions regarding code, make sure your environment matches the following
+requirements:
+
+* composer is executable
+
+* PHP on CLI is executable
+
+* MySQL is up and running with user *dev* and password *dev* on *127.0.0.1* or to overwrite the
+  environment variables, see :file:`Makefile`.
+  And MySQL is not set to strict mode as TYPO3 doesn't support strict mode, see
+  https://review.typo3.org/#/c/26725/3/INSTALL.md.
+
+* Elasticsearch is installed and up and running on *localhost:9200*.
+
+Then setup your system::
+
+    git clone git@github.com:codappix/search_core.git \
+        && cd search_core \
+        && export typo3DatabaseName="searchcoretest87" \
+        && export TYPO3_VERSION="~8.7" \
+        && make install \
+        && make unitTests \
+        && make functionalTests
+
+If all tests are okay, start your work.
+
+If you are working with multiple TYPO3 versions make sure to export `typo3DatabaseName` and
+`TYPO3_VERSION` in your environment like::
+
+    export typo3DatabaseName="searchcoretest76"
+    export TYPO3_VERSION="~7.6"
+
+Also run the install command for each version before running any tests. Only this will make sure you
+are testing against the actual TYPO3 Version and database scheme.
+
+.. _contribution_development:
+
+Development
+-----------
+
+All changes are introduced through pull requests at `Github`_ and should contain the following:
+
+* Adjusted tests if tests existed before. Otherwise they will break on `travis-ci`_.
+
+* New tests whenever possible and useful.
+
+* Code has to follow `PSR-2`_.
+
+* Adjusted documentation.
+
+* Make sure to follow the documented :ref:`concepts`.
+
+.. _Github: https://github.com/codappix/search_core
+.. _travis-ci: https://travis-ci.org/
+.. _PSR-2: http://www.php-fig.org/psr/psr-2/

Documentation/source/development/dataProcessor.rst

@@ -0,0 +1,15 @@
+.. _development_dataprocessor:
+
+Develop a new DataProcessor
+===========================
+
+Make sure you understood :ref:`concepts_dataprocessing`.
+
+Each DataProcessor has to be a single class which implements
+``Codappix\SearchCore\DataProcessing\ProcessorInterface``.
+
+Make sure you support both, Frontend and Backend, as processors can be called during searching and
+indexing. Therefore do not rely on e.g. ``TSFE``, make sure dependencies are met and your code will
+work in both environments.
+
+Dependency Injection is working for custom DataProcessors.

Documentation/source/development/indexer.rst

@@ -0,0 +1,21 @@
+.. _development_indexer:
+
+Develop a new Indexer
+=====================
+
+Make sure you understood :ref:`concepts_indexing`.
+
+Each indexer has to be a single class which implements
+``Codappix\SearchCore\Domain\Index\IndexerInterface``.
+
+The indexer should call the connection with all necessary information about the document(s) to
+trigger indexing or deletion of whole index.
+
+As this is the "indexer", deletion of single documents is directly processed by the connection.
+
+``setIdentifier`` is called with the identifier of the current Indexer. This might be useful to
+fetch configuration, related to the indexing, from
+``Codappix\SearchCore\Configuration\ConfigurationContainerInterface``.
+
+Dependency Injection is working for custom indexers, therefore you are able to inject the
+``ConfigurationContainerInterface``.

Documentation/source/features.rst

@@ -0,0 +1,62 @@
+.. _features:
+
+Features
+========
+
+The following features are available:
+
+.. _features_indexing:
+
+Indexing
+--------
+
+Indexing of data is possible. We deliver an indexer for TCA with zero configuration needs. You can
+also provide custom indexer for arbitrary data which is not indexable through TCA.
+
+Also a finisher for TYPO3 Form-Extension is provided to integrate indexing after something was
+update through the Form-Extension.
+
+Indexing is done through Hooks and CLI. We therefore provide commands to index and delete indexed
+data.
+
+.. _features_search:
+
+Searching
+---------
+
+.. note::
+    Currently only integrated for Elasticsearch with no abstraction.
+    If you need to implement your own search, please open an issue on Github and we will change the code
+    base.
+
+Via TypoScript it's possible to configure the fields to query, minimum match and script fields.
+Also multiple filter are supported, filtering results by fields.
+
+Facets / aggregates are also possible. Therefore a mapping has to be defined in TypoScript for
+indexing, and the facets itself while searching.
+
+.. _features_dataProcessing:
+
+DataProcessing
+--------------
+
+DataProcessing, as known from ``FLUIDTEMPLATE``, is available while indexing and for search results.
+Each record and result item can be processed by multiple processor to prepare data for indexing and
+output.
+
+See :ref:`concepts_dataprocessing` in :ref:`concepts` section.
+
+.. _features_planned:
+
+Planned
+-------
+
+The following features are currently planned and will be integrated:
+
+#. :issue:`25` Multi language.
+#. :issue:`94` Respect access rights while indexing relations.
+#. :issue:`75` Configuration of index name (for Elasticsearch).
+
+For a full list, check out our `open issues`_.
+
+.. _open issues: https://github.com/Codappix/search_core/issues

Documentation/source/index.rst

@@ -1,16 +1,19 @@
 .. include:: readme.rst
 
 Table of Contents
-=================
+-----------------
 
 .. toctree::
    :maxdepth: 1
    :glob:
 
+   features
    installation
    configuration
    usage
    concepts
    connections
    indexer
+   dataprocessors
    development
+   changelog

Documentation/source/indexer.rst

@@ -5,6 +5,8 @@ Indexer
 
 See Concept of :ref:`concepts_indexing` for further background information.
 
+For information about implementing a new indexer, take a look at :ref:`development_indexer`.
+
 The extension provides the following indexer out of the box:
 
 .. _TcaIndexer:
@@ -15,19 +17,24 @@ TcaIndexer
 Provides zero configuration TYPO3 integration by using the :ref:`t3tcaref:start`. You just can
 start indexing TYPO3.
 
+Just add the indexer for a TYPO3 table. The indexer will use the TCA to fetch all necessary
+information like relations.
+
+.. note::
+
+  Not all relations are resolved yet, see :issue:`17`.
+
+.. _PagesIndexer:
+
+PagesIndexer
+------------
+
+Provides zero configuration TYPO3 integration by using the :ref:`t3tcaref:start`. You just can
+start indexing TYPO3.
+
 The indexer will use the TCA to fetch all necessary information like relations. Currently the
-implementation is very basic. In future it will also provide mapping for :ref:`Elasticsearch` and
+implementation is very basic.
-further stuff.
-
-The indexer is configurable through the following options:
-
-* :ref:`allowedTables`
-
-* :ref:`rootLineBlacklist`
-
-* :ref:`additionalWhereClause`
 
 .. note::
 
   Not all relations are resolved yet, see :issue:`17` and :pr:`20`.
-  Also the `pages`-Table is not available yet, see :issue:`24`.

Documentation/source/installation.rst

@@ -4,20 +4,41 @@
 Installation
 ============
 
+Composer
+--------
+
 The extension can be installed through composer::
 
-    composer require "leonmrni/search_core dev-feature/integrate-elasticsearch"
+    composer require "codappix/search_core" "~1.0.0"
 
-or by `downloading`_ and placing it inside the :file:`typo3conf/ext`-Folder of your installation.
+Note that you have to allow unstable packages:
-In that case you need to install all dependencies yourself. Dependencies are:
+
+.. code-block:: json
+
+   {
+      "minimum-stability": "dev",
+      "prefer-stable": true
+   }
+
+Download
+--------
+
+You can also `download`_ the extension and placing it inside the :file:`typo3conf/ext`-Folder of
+your installation.  In that case you need to install all dependencies yourself. Dependencies are:
 
 .. literalinclude:: ../../composer.json
    :caption: Dependencies from composer.json
    :lines: 19-21
    :dedent: 8
 
+Setup
+-----
 
 Afterwards you need to enable the extension through the extension manager and include the static
-typoscript setup.
+TypoScript setup.
 
-.. _downloading: https://github.com/DanielSiepmann/search_core/archive/feature/integrate-elasticsearch.zip
+If you **don't** want to use the included Elasticsearch integration, you have to disable it in the
+extension manager configuration of the extension by checking the checkbox.
+It's currently enabled by default but will be moved into its own extension in the future.
+
+.. _download: https://github.com/codappix/search_core/archive/develop.zip

Documentation/source/readme.rst

@@ -1,5 +1,5 @@
-TYPO3 Extension search_core's documentation!
+TYPO3 Extension search_core
-============================================
+===========================
 
 Introduction
 ============
@@ -8,23 +8,21 @@ What does it do?
 ----------------
 
 The goal of this extension is to provide search integrations into TYPO3 CMS. The extension will
-abstract the concrete implementations to allow exchange of concrete backends like Elasticsearch or
+provide a convenient API to allow developers to provide concrete implementations of backends like
-solr.
+Elasticsearch, Algolia or Solr.
 
 The extension provides integration into TYPO3 like a frontend plugin for searches and hooks to
 update search indexes on updates. Also a command line interface is provided for interactions like
-reindexing.
+re-indexing.
 
 Current state
 -------------
 
-This is still a very early alpha version. More information can be taken from Github at
+This is still a very early beta version. More information can be taken from Github at
-`current issues`_ and `current projects`_.
+`current issues`_.
 
-We are also focusing on Code Quality and Testing through `travis ci`_, `scrutinizer`_ and `codacy`_.
+We are also focusing on Code Quality and Testing through `travis ci`_, ``phpcs``, ``phpunit`` and
+``phpstan``.
 
 .. _current issues: https://github.com/Codappix/search_core/issues
-.. _current projects: https://github.com/Codappix/search_core/projects
 .. _travis ci: https://travis-ci.org/Codappix/search_core
-.. _scrutinizer: https://scrutinizer-ci.com/g/Codappix/search_core/inspections
-.. _codacy: https://www.codacy.com/app/Codappix/search_core/dashboard

Documentation/source/usage.rst

@@ -11,12 +11,27 @@ Manual indexing
 
 You can trigger indexing from CLI::
 
-    ./typo3/cli_dispatch.phpsh extbase index:index --table 'tt_content'
+    ./typo3/cli_dispatch.phpsh extbase index:index --identifier 'pages'
+    ./bin/typo3cms index:index --identifier 'pages'
 
-This will index the table ``tt_content`` using the :ref:`TcaIndexer`.
+This will index the table ``pages`` using the :ref:`TcaIndexer`.
 
-Only one table per call is available, to index multiple tables just make multiple calls.
+Only one index per call is available, to run multiple indexers, just make multiple calls.
-The tables have to be white listed through :ref:`allowedTables` option.
+The indexers have to be defined in TypoScript via :ref:`configuration_options_index`.
+
+.. _usage_manual_deletion:
+
+Manual deletion
+---------------
+
+You can trigger deletion for a single index from CLI::
+
+    ./typo3/cli_dispatch.phpsh extbase index:delete --identifier 'pages'
+    ./bin/typo3cms index:delete --identifier 'pages'
+
+This will delete the index for the table ``pages``.
+
+Only one delete per call is available, to run multiple deletions, just make multiple calls.
 
 .. _usage_auto_indexing:
 
@@ -24,12 +39,39 @@ Auto indexing
 -------------
 
 Indexing is done through hooks every time an record is changed.
-The tables have to be white listed through :ref:`allowedTables` option.
+The tables have to be configured via :ref:`configuration_options_index`.
 
 .. note::
 
   Not all hook operations are supported yet, see :issue:`27`.
 
+.. _usage_form_finisher:
+
+Form finisher
+-------------
+
+A form finisher is provided to integrate indexing into form extension.
+
+Add form finisher to your available finishers and configure it like:
+
+.. code-block:: yaml
+
+    -
+        identifier: SearchCoreIndexer
+        options:
+            action: 'delete'
+            indexIdentifier: 'fe_users'
+            recordUid: '{FeUser.user.uid}'
+
+All three options are necessary, where:
+
+``action``
+    Is one of ``delete``, ``update`` or ``add``.
+``indexIdentifier``
+    Is a configured index identifier.
+``recordUid``
+    Has to be the uid of the record to index.
+
 .. _usage_searching:
 
 Searching / Frontend Plugin
@@ -37,3 +79,47 @@ Searching / Frontend Plugin
 
 To provide a search interface you can insert the frontend Plugin as normal content element of type
 plugin. The plugin is named *Search Core*.
+
+Please provide your own template, the extension will not deliver a useful template for now.
+
+The Extbase mapping is used, this way you can create a form:
+
+.. code-block:: html
+
+   <f:form name="searchRequest" object="{searchRequest}">
+       <f:form.textfield property="query" />
+       <f:form.submit value="search" />
+   </f:form>
+
+.. _usage_searching_filter:
+
+Filter
+""""""
+
+Thanks to Extbase mapping, filter are added to the form:
+
+.. code-block:: html
+
+   <f:form.textfield property="filter.exampleName" value="the value to match" />
+
+.. _usage_searching_facets:
+
+Facets
+""""""
+
+To add a facet as criteria for searching, use :ref:`usage_searching_filter`.
+
+To display facet results use:
+
+.. code-block:: html
+
+    <f:for each="{searchResult.facets}" as="facet">
+        <f:for each="{facet.options}" as="option">
+            <label for="{option.name}-desktop">
+                <f:form.checkbox value="{option.name}" property="filter.{facet.field}" />
+                {f:translate(id: 'search.filter.channel.{option.name}', default: option.name, extensionName: 'SitePackage')}
+                ({option.count})
+            </label>
+        </f:for>
+    </f:for>
+

Makefile

@@ -1,19 +1,32 @@
 mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST)))
 current_dir := $(dir $(mkfile_path))
 
-TYPO3_WEB_DIR := $(current_dir).Build/Web
+TYPO3_WEB_DIR := $(current_dir).Build/web
+TYPO3_PATH_ROOT := $(current_dir).Build/web
 # Allow different versions on travis
-TYPO3_VERSION ?= ~7.6
+TYPO3_VERSION ?= ~8.7
 typo3DatabaseName ?= "searchcore_test"
 typo3DatabaseUsername ?= "dev"
 typo3DatabasePassword ?= "dev"
 typo3DatabaseHost ?= "127.0.0.1"
 
+sourceOrDist=--prefer-dist
+ifeq ($(TYPO3_VERSION),~7.6)
+	sourceOrDist=--prefer-source
+endif
+
 .PHONY: install
 install: clean
-	COMPOSER_PROCESS_TIMEOUT=1000 composer require -vv --dev --prefer-source --ignore-platform-reqs typo3/cms="$(TYPO3_VERSION)"
+	if [ $(TYPO3_VERSION) = ~7.6 ]; then \
+		patch composer.json Tests/InstallPatches/composer.json.patch; \
+	fi
+
+	COMPOSER_PROCESS_TIMEOUT=1000 composer require -vv --dev $(sourceOrDist) typo3/cms="$(TYPO3_VERSION)"
 	git checkout composer.json
 
+cgl:
+	./.Build/bin/phpcs
+
 functionalTests:
 	typo3DatabaseName=$(typo3DatabaseName) \
 		typo3DatabaseUsername=$(typo3DatabaseUsername) \
@@ -23,6 +36,11 @@ functionalTests:
 		.Build/bin/phpunit --colors --debug -v \
 			-c Tests/Functional/FunctionalTests.xml
 
+unitTests:
+	TYPO3_PATH_WEB=$(TYPO3_WEB_DIR) \
+		.Build/bin/phpunit --colors --debug -v \
+		-c Tests/Unit/UnitTests.xml
+
 uploadCodeCoverage: uploadCodeCoverageToScrutinizer uploadCodeCoverageToCodacy
 
 uploadCodeCoverageToScrutinizer:

Tests/Functional/AbstractFunctionalTestCase.php

@@ -66,4 +66,9 @@ abstract class AbstractFunctionalTestCase extends CoreTestCase
     {
         return ['EXT:search_core/Tests/Functional/Fixtures/BasicSetup.ts'];
     }
+
+    protected function isLegacyVersion() : bool
+    {
+        return \TYPO3\CMS\Core\Utility\VersionNumberUtility::convertVersionNumberToInteger(TYPO3_version) < 8000000;
+    }
 }

Tests/Functional/Bootstrap.php

@@ -0,0 +1,9 @@
+<?php
+
+$filePath = '.Build/vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTestsBootstrap.php';
+
+if (getenv('TYPO3_VERSION') === '~7.6') {
+    $filePath = '.Build/vendor/typo3/cms/typo3/sysext/core/Build/FunctionalTestsBootstrap.php';
+}
+
+require_once dirname(dirname(__DIR__)) . '/' . $filePath;

Tests/Functional/Connection/Elasticsearch/AbstractFunctionalTestCase.php

@@ -43,11 +43,19 @@ abstract class AbstractFunctionalTestCase extends BaseFunctionalTestCase
             'host' => getenv('ES_HOST') ?: \Elastica\Connection::DEFAULT_HOST,
             'port' => getenv('ES_PORT') ?: \Elastica\Connection::DEFAULT_PORT,
         ]);
+
+        // Start with clean system for test.
+        $this->cleanUp();
     }
 
     public function tearDown()
     {
-        // Delete everything so next test starts clean.
+        // Make system clean again.
+        $this->cleanUp();
+    }
+
+    protected function cleanUp()
+    {
         $this->client->getIndex('_all')->delete();
         $this->client->getIndex('_all')->clearCache();
     }

Tests/Functional/Connection/Elasticsearch/FacetTest.php

@@ -0,0 +1,78 @@
+<?php
+namespace Codappix\SearchCore\Tests\Functional\Connection\Elasticsearch;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\SearchCore\Domain\Index\IndexerFactory;
+use Codappix\SearchCore\Domain\Model\SearchRequest;
+use Codappix\SearchCore\Domain\Search\SearchService;
+use TYPO3\CMS\Extbase\Object\ObjectManager;
+
+class FacetTest extends AbstractFunctionalTestCase
+{
+    protected function getTypoScriptFilesForFrontendRootPage()
+    {
+        return array_merge(
+            parent::getTypoScriptFilesForFrontendRootPage(),
+            ['EXT:search_core/Tests/Functional/Fixtures/Searching/Facet.ts']
+        );
+    }
+
+    protected function getDataSets()
+    {
+        return array_merge(
+            parent::getDataSets(),
+            ['Tests/Functional/Fixtures/Searching/Filter.xml']
+        );
+    }
+
+    /**
+     * @test
+     */
+    public function itsPossibleToFetchFacetsForField()
+    {
+        \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class)
+            ->get(IndexerFactory::class)
+            ->getIndexer('tt_content')
+            ->indexAllDocuments()
+            ;
+
+        $searchService = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class)
+            ->get(SearchService::class);
+
+        $searchRequest = new SearchRequest();
+        $result = $searchService->search($searchRequest);
+
+        $this->assertSame(1, count($result->getFacets()), 'Did not receive the single defined facet.');
+
+        $facet = current($result->getFacets());
+        $this->assertSame('contentTypes', $facet->getName(), 'Name of facet was not as expected.');
+        $this->assertSame('CType', $facet->getField(), 'Field of facet was not expected.');
+
+        $options = $facet->getOptions();
+        $this->assertSame(2, count($options), 'Did not receive the expected number of possible options for facet.');
+        $option = $options['HTML'];
+        $this->assertSame('HTML', $option->getName(), 'Option did not have expected Name.');
+        $this->assertSame(1, $option->getCount(), 'Option did not have expected count.');
+        $option = $options['Header'];
+        $this->assertSame('Header', $option->getName(), 'Option did not have expected Name.');
+        $this->assertSame(1, $option->getCount(), 'Option did not have expected count.');
+    }
+}

Tests/Functional/Connection/Elasticsearch/FilterTest.php

@@ -0,0 +1,61 @@
+<?php
+namespace Codappix\SearchCore\Tests\Functional\Connection\Elasticsearch;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\SearchCore\Domain\Index\IndexerFactory;
+use Codappix\SearchCore\Domain\Model\SearchRequest;
+use Codappix\SearchCore\Domain\Search\SearchService;
+use TYPO3\CMS\Extbase\Object\ObjectManager;
+
+class FilterTest extends AbstractFunctionalTestCase
+{
+    protected function getDataSets()
+    {
+        return array_merge(
+            parent::getDataSets(),
+            ['Tests/Functional/Fixtures/Searching/Filter.xml']
+        );
+    }
+
+    /**
+     * @test
+     */
+    public function itsPossibleToFilterResultsByASingleField()
+    {
+        \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class)
+            ->get(IndexerFactory::class)
+            ->getIndexer('tt_content')
+            ->indexAllDocuments()
+            ;
+
+        $searchService = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class)
+            ->get(SearchService::class);
+        $searchRequest = new SearchRequest('Search Word');
+
+        $result = $searchService->search($searchRequest);
+        $this->assertSame(2, count($result), 'Did not receive both indexed elements without filter.');
+
+        $searchRequest->setFilter(['CType' => 'HTML']);
+        $result = $searchService->search($searchRequest);
+        $this->assertSame(5, (int) $result->getResults()[0]['uid'], 'Did not get the expected result entry.');
+        $this->assertSame(1, count($result), 'Did not receive the single filtered element.');
+    }
+}

Tests/Functional/Connection/Elasticsearch/IndexDeletionTest.php

@@ -0,0 +1,50 @@
+<?php
+namespace Codappix\SearchCore\Tests\Functional\Connection\Elasticsearch;
+
+/*
+ * Copyright (C) 2016  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\SearchCore\Domain\Index\IndexerFactory;
+use TYPO3\CMS\Extbase\Object\ObjectManager;
+
+class IndexDeletionTest extends AbstractFunctionalTestCase
+{
+    /**
+     * @test
+     */
+    public function indexIsDeleted()
+    {
+        $this->client->getIndex('typo3content')->create();
+        $this->assertTrue(
+            $this->client->getIndex('typo3content')->exists(),
+            'Could not create index for test.'
+        );
+
+        \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class)
+            ->get(IndexerFactory::class)
+            ->getIndexer('tt_content')
+            ->delete()
+            ;
+
+        $this->assertFalse(
+            $this->client->getIndex('typo3content')->exists(),
+            'Index could not be deleted through command controller.'
+        );
+    }
+}

Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php

@@ -49,6 +49,29 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
 
         $response = $this->client->request('typo3content/_search?q=*:*');
 
+        $this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.');
+        $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.');
+        $this->assertArraySubset(
+            ['_source' => ['header' => 'indexed content element']],
+            $response->getData()['hits']['hits'][1],
+            false,
+            'Record was not indexed.'
+        );
+    }
+
+    /**
+     * @test
+     */
+    public function indexSingleBasicTtContent()
+    {
+        \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class)
+            ->get(IndexerFactory::class)
+            ->getIndexer('tt_content')
+            ->indexDocument(6)
+            ;
+
+        $response = $this->client->request('typo3content/_search?q=*:*');
+
         $this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.');
         $this->assertSame($response->getData()['hits']['total'], 1, 'Not exactly 1 document was indexed.');
         $this->assertArraySubset(
@@ -90,7 +113,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
         $response = $this->client->request('typo3content/_search?q=*:*');
 
         $this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.');
-        $this->assertSame($response->getData()['hits']['total'], 1, 'Not exactly 1 document was indexed.');
+        $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.');
     }
 
     /**
@@ -113,16 +136,18 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
         $response = $this->client->request('typo3content/_search?q=*:*');
 
         $this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.');
-        $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.');
+        $this->assertSame($response->getData()['hits']['total'], 3, 'Not exactly 3 documents were indexed.');
+        $response = $this->client->request('typo3content/_search?q=uid:11');
         $this->assertArraySubset(
             ['_source' => ['header' => 'Also indexable record']],
             $response->getData()['hits']['hits'][0],
             false,
             'Record was not indexed.'
         );
+        $response = $this->client->request('typo3content/_search?q=uid:6');
         $this->assertArraySubset(
             ['_source' => ['header' => 'indexed content element']],
-            $response->getData()['hits']['hits'][1],
+            $response->getData()['hits']['hits'][0],
             false,
             'Record was not indexed.'
         );
@@ -143,24 +168,24 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
 
         $response = $this->client->request('typo3content/_search?q=*:*');
         $this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.');
-        $this->assertSame($response->getData()['hits']['total'], 3, 'Not exactly 3 documents were indexed.');
+        $this->assertSame($response->getData()['hits']['total'], 4, 'Not exactly 4 documents were indexed.');
 
-        $response = $this->client->request('typo3content/_search?q=uid:9');
+        $response = $this->client->request('typo3content/_search?q=uid:11');
         $this->assertArraySubset(
             ['_source' => [
-                'uid' => '9',
+                'uid' => '11',
                 'CType' => 'Header', // Testing items
-                'categories' => ['Category 1', 'Category 2'], // Testing mm (with sorting)
+                'categories' => ['Category 2', 'Category 1'], // Testing mm
             ]],
             $response->getData()['hits']['hits'][0],
             false,
             'Record was not indexed with resolved category relations to multiple values.'
         );
 
-        $response = $this->client->request('typo3content/_search?q=uid:10');
+        $response = $this->client->request('typo3content/_search?q=uid:12');
         $this->assertArraySubset(
             ['_source' => [
-                'uid' => '10',
+                'uid' => '12',
                 'CType' => 'Header',
                 'categories' => ['Category 2'],
             ]],
@@ -180,4 +205,40 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
             'Record was indexed with resolved category relation, but should not have any.'
         );
     }
+
+    /**
+     * @test
+     */
+    public function indexingDeltedRecordIfRecordShouldBeIndexedButIsNoLongerAvailableAndWasAlreadyIndexed()
+    {
+        \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class)
+            ->get(IndexerFactory::class)
+            ->getIndexer('tt_content')
+            ->indexAllDocuments()
+            ;
+
+        $response = $this->client->request('typo3content/_search?q=*:*');
+        $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.');
+
+        if ($this->isLegacyVersion()) {
+            $this->getDatabaseConnection()
+                ->exec_UPDATEquery('tt_content', 'uid = 10', ['hidden' => 1]);
+        } else {
+            $this->getConnectionPool()->getConnectionForTable('tt_content')
+                ->update(
+                    'tt_content',
+                    ['hidden' => true],
+                    ['uid' => 10]
+                );
+        }
+
+        \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class)
+            ->get(IndexerFactory::class)
+            ->getIndexer('tt_content')
+            ->indexDocument(10)
+            ;
+
+        $response = $this->client->request('typo3content/_search?q=*:*');
+        $this->assertSame($response->getData()['hits']['total'], 1, 'Not exactly 1 document is in index.');
+    }
 }

Tests/Functional/DataProcessing/ContentObjectDataProcessorAdapterProcessorTest.php

@@ -0,0 +1,62 @@
+<?php
+namespace Codappix\SearchCore\Tests\Functional\DataProcessing;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\SearchCore\Compatibility\TypoScriptService76;
+use Codappix\SearchCore\Compatibility\TypoScriptService;
+use Codappix\SearchCore\DataProcessing\ContentObjectDataProcessorAdapterProcessor;
+use Codappix\SearchCore\Tests\Functional\AbstractFunctionalTestCase;
+use TYPO3\CMS\Frontend\DataProcessing\SplitProcessor;
+
+class ContentObjectDataProcessorAdapterProcessorTest extends AbstractFunctionalTestCase
+{
+    /**
+     * @test
+     */
+    public function contentObjectDataProcessorIsExecuted()
+    {
+        $record = ['content' => 'value1, value2'];
+        $configuration = [
+            '_dataProcessor' => SplitProcessor::class,
+            'delimiter' => ',',
+            'fieldName' => 'content',
+            'as' => 'new_content',
+        ];
+        $expectedData = [
+            'content' => 'value1, value2',
+            'new_content' => ['value1', 'value2'],
+        ];
+
+        if ($this->isLegacyVersion()) {
+            $typoScriptService = new TypoScriptService76();
+        } else {
+            $typoScriptService = new TypoScriptService();
+        }
+
+        $subject = new ContentObjectDataProcessorAdapterProcessor($typoScriptService);
+        $processedData = $subject->processData($record, $configuration);
+        $this->assertSame(
+            $expectedData,
+            $processedData,
+            'The processor did not return the expected processed record.'
+        );
+    }
+}

Tests/Functional/Fixtures/BasicSetup.ts

@@ -8,9 +8,37 @@ plugin {
                 }
             }
 
-            indexer {
+            indexing {
-                tca {
+                tt_content {
-                    allowedTables = tt_content
+                    indexer = Codappix\SearchCore\Domain\Index\TcaIndexer
+
+                    additionalWhereClause (
+                        tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu', 'shortcut', 'search', 'login')
+                        AND tt_content.bodytext != ''
+                    )
+
+                    mapping {
+                        CType {
+                            type = keyword
+                        }
+                    }
+                }
+
+                pages {
+                    indexer = Codappix\SearchCore\Domain\Index\TcaIndexer\PagesIndexer
+                    abstractFields = abstract, description, bodytext
+
+                    mapping {
+                        CType {
+                            type = keyword
+                        }
+                    }
+                }
+            }
+
+            searching {
+                fields {
+                    query = _all
                 }
             }
         }

Tests/Functional/Fixtures/BasicSetup.xml

@@ -4,5 +4,6 @@
         <uid>1</uid>
         <pid>0</pid>
         <title>Root page containing necessary TypoScript</title>
+        <description>Used as abstract as no abstract is defined.</description>
     </pages>
 </dataset>

Tests/Functional/Fixtures/Indexing/IndexTcaTable.xml

@@ -60,4 +60,43 @@
         <filelink_sorting>0</filelink_sorting>
     </tt_content>
 
+    <tt_content>
+        <uid>9</uid>
+        <pid>1</pid>
+        <tstamp>1480686370</tstamp>
+        <crdate>1480686370</crdate>
+        <hidden>0</hidden>
+        <sorting>72</sorting>
+        <CType>div</CType>
+        <header>not indexed due to ctype</header>
+        <bodytext>this is the content of div content element that should not get indexed</bodytext>
+        <media>0</media>
+        <layout>0</layout>
+        <deleted>0</deleted>
+        <cols>0</cols>
+        <starttime>0</starttime>
+        <endtime>0</endtime>
+        <colPos>0</colPos>
+        <filelink_sorting>0</filelink_sorting>
+    </tt_content>
+
+    <tt_content>
+        <uid>10</uid>
+        <pid>1</pid>
+        <tstamp>1480686370</tstamp>
+        <crdate>1480686370</crdate>
+        <hidden>0</hidden>
+        <sorting>72</sorting>
+        <CType>html</CType>
+        <header>Indexed without html tags</header>
+        <bodytext><![CDATA[<p>Some text in paragraph</p>]]></bodytext>
+        <media>0</media>
+        <layout>0</layout>
+        <deleted>0</deleted>
+        <cols>0</cols>
+        <starttime>0</starttime>
+        <endtime>0</endtime>
+        <colPos>0</colPos>
+        <filelink_sorting>0</filelink_sorting>
+    </tt_content>
 </dataset>