Merge branch 'develop' into feature/75-make-index-configurable

# Conflicts:
#	Documentation/source/configuration/connections.rst
#	Documentation/source/connections.rst

.phan/config.php

@@ -0,0 +1,96 @@
+<?php
+
+/**
+ * This configuration will be read and overlaid on top of the
+ * default configuration. Command line arguments will be applied
+ * after this file is read.
+ */
+return [
+
+    // Supported values: '7.0', '7.1', '7.2', null.
+    // If this is set to null,
+    // then Phan assumes the PHP version which is closest to the minor version
+    // of the php executable used to execute phan.
+    "target_php_version" => '7.0',
+
+    // Override to hardcode existence and types of (non-builtin) globals.
+    // Class names should be prefixed with '\\'.
+    // (E.g. ['_FOO' => '\\FooClass', 'page' => '\\PageClass', 'userId' => 'int'])
+    'globals_type_map' => [
+        '_EXTKEY' => 'string',
+        'EM_CONF' => 'array',
+    ],
+
+    // A list of directories that should be parsed for class and
+    // method information. After excluding the directories
+    // defined in exclude_analysis_directory_list, the remaining
+    // files will be statically analyzed for errors.
+    //
+    // Thus, both first-party and third-party code being used by
+    // your application should be included in this list.
+    'directory_list' => [
+        'Classes',
+        '.Build/vendor',
+    ],
+
+    // A list of files to include in analysis
+    'file_list' => [
+        'ext_emconf.php',
+        'ext_tables.php',
+        'ext_localconf.php',
+    ],
+
+    // A directory list that defines files that will be excluded
+    // from static analysis, but whose class and method
+    // information should be included.
+    //
+    // Generally, you'll want to include the directories for
+    // third-party code (such as "vendor/") in this list.
+    //
+    // n.b.: If you'd like to parse but not analyze 3rd
+    //       party code, directories containing that code
+    //       should be added to the `directory_list` as
+    //       to `exclude_analysis_directory_list`.
+    "exclude_analysis_directory_list" => [
+        '.Build/vendor'
+    ],
+
+    // A list of directories that should be parsed for class and
+    // method information. After excluding the directories
+    // defined in exclude_analysis_directory_list, the remaining
+    // files will be statically analyzed for errors.
+    //
+    // Thus, both first-party and third-party code being used by
+    // your application should be included in this list.
+    'directory_list' => [
+        'Classes',
+        // 'Tests',
+        '.Build/vendor',
+    ],
+
+    // The number of processes to fork off during the analysis phase.
+    'processes' => 3,
+
+    // Add any issue types (such as 'PhanUndeclaredMethod')
+    // here to inhibit them from being reported
+    'suppress_issue_types' => [
+        'PhanDeprecatedFunction', // For now
+        'PhanParamTooMany', // For now, due to ObjectManager->get()
+    ],
+
+    // A list of plugin files to execute.
+    // See https://github.com/phan/phan/tree/master/.phan/plugins for even more.
+    // (Pass these in as relative paths.
+    // The 0.10.2 release will allow passing 'AlwaysReturnPlugin' if referring to a plugin that is bundled with Phan)
+    'plugins' => [
+        // checks if a function, closure or method unconditionally returns.
+        'AlwaysReturnPlugin',  // can also be written as 'vendor/phan/phan/.phan/plugins/AlwaysReturnPlugin.php'
+        // Checks for syntactically unreachable statements in
+        // the global scope or function bodies.
+        'UnreachableCodePlugin',
+        'DollarDollarPlugin',
+        'DuplicateArrayKeyPlugin',
+        'PregRegexCheckerPlugin',
+        'PrintfCheckerPlugin',
+    ],
+];

.scrutinizer.yml

@@ -1,3 +1,11 @@
+build:
+    nodes:
+        analysis:
+            project_setup:
+                override: true
+            tests:
+                override: [php-scrutinizer-run]
+
 filter:
     excluded_paths:
         - 'Configuration/*'
@@ -19,7 +27,7 @@ tools:
     php_hhvm:
         enabled: true
         config:
-            use_undeclared_constant:  false
+            use_undeclared_constant: false
 
     php_mess_detector:
         enabled: true
@@ -34,5 +42,5 @@ tools:
         enabled: true
     # We generate code coverage during tests at travis and will send them here
     external_code_coverage:
-      runs: 2
-      timeout: 1200
+        runs: 2
+        timeout: 1200

.travis.yml

@@ -5,13 +5,15 @@ addons:
     packages:
       - oracle-java8-set-default
 before_install:
-  - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.2.0.deb && sudo dpkg -i --force-confnew elasticsearch-5.2.0.deb && sudo service elasticsearch start
+  - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.2.0.deb && sudo dpkg -i --force-confnew elasticsearch-5.2.0.deb && sudo service elasticsearch restart
   - mysql -u root -e 'GRANT ALL ON `typo3_ci_ft%`.* TO travis@127.0.0.1;'
 
 language: php
 
 php:
+  - 7.0
   - 7.1
+  - 7.2
 
 env:
   global:
@@ -23,6 +25,9 @@ env:
     - typo3DatabaseHost="127.0.0.1"
     - typo3DatabaseUsername="travis"
     - typo3DatabasePassword=""
+  matrix:
+    - TYPO3_VERSION="~7.6"
+    - TYPO3_VERSION="~8.7"
 
 matrix:
   fast_finish: true
@@ -33,6 +38,7 @@ services:
 install: make install
 
 script:
+  - make cgl
   - make unitTests
   - make functionalTests
 

Classes/Command/IndexCommandController.php

@@ -22,7 +22,6 @@ namespace Codappix\SearchCore\Command;
 
 use Codappix\SearchCore\Domain\Index\IndexerFactory;
 use Codappix\SearchCore\Domain\Index\NoMatchingIndexerException;
-use TYPO3\CMS\Core\Utility\GeneralUtility;
 use TYPO3\CMS\Extbase\Mvc\Controller\CommandController;
 
 /**
@@ -48,7 +47,7 @@ class IndexCommandController extends CommandController
      *
      * @param string $identifier
      */
-    public function indexCommand($identifier)
+    public function indexCommand(string $identifier)
     {
         try {
             $this->indexerFactory->getIndexer($identifier)->indexAllDocuments();
@@ -63,7 +62,7 @@ class IndexCommandController extends CommandController
      *
      * @param string $identifier
      */
-    public function deleteCommand($identifier)
+    public function deleteCommand(string $identifier)
     {
         try {
             $this->indexerFactory->getIndexer($identifier)->delete();

Classes/Compatibility/ImplementationRegistrationService.php

@@ -0,0 +1,56 @@
+<?php
+namespace Codappix\SearchCore\Compatibility;
+
+/*
+ * Copyright (C) 2018  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Core\Utility\VersionNumberUtility;
+use TYPO3\CMS\Extbase\Object\Container\Container;
+
+/**
+ * Register different concrete implementations, depending on current TYPO3 version.
+ * This way we can provide working implementations for multiple TYPO3 versions.
+ */
+class ImplementationRegistrationService
+{
+    public static function registerImplementations()
+    {
+        $container = GeneralUtility::makeInstance(Container::class);
+        if (VersionNumberUtility::convertVersionNumberToInteger(TYPO3_version) >= 8000000) {
+            $container->registerImplementation(
+                \Codappix\SearchCore\Compatibility\TypoScriptServiceInterface::class,
+                \Codappix\SearchCore\Compatibility\TypoScriptService::class
+            );
+            $container->registerImplementation(
+                \Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface::class,
+                \Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService::class
+            );
+        } else {
+            $container->registerImplementation(
+                \Codappix\SearchCore\Compatibility\TypoScriptServiceInterface::class,
+                \Codappix\SearchCore\Compatibility\TypoScriptService76::class
+            );
+            $container->registerImplementation(
+                \Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface::class,
+                \Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService76::class
+            );
+        }
+    }
+}

Classes/Compatibility/TypoScriptService.php

@@ -0,0 +1,31 @@
+<?php
+namespace Codappix\SearchCore\Compatibility;
+
+/*
+ * Copyright (C) 2018  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use TYPO3\CMS\Core\TypoScript\TypoScriptService as CoreTypoScriptService;
+
+/**
+ * Used since TYPO3 CMS 8.7.
+ */
+class TypoScriptService extends CoreTypoScriptService implements TypoScriptServiceInterface
+{
+
+}

Classes/Compatibility/TypoScriptService76.php

@@ -0,0 +1,31 @@
+<?php
+namespace Codappix\SearchCore\Compatibility;
+
+/*
+ * Copyright (C) 2018  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use TYPO3\CMS\Extbase\Service\TypoScriptService as CoreTypoScriptService;
+
+/**
+ * Used before TYPO3 CMS 8.7.
+ */
+class TypoScriptService76 extends CoreTypoScriptService implements TypoScriptServiceInterface
+{
+
+}

Classes/Compatibility/TypoScriptServiceInterface.php

@@ -0,0 +1,30 @@
+<?php
+namespace Codappix\SearchCore\Compatibility;
+
+/*
+ * Copyright (C) 2018  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * Allows to use DI configuration to switch concrete implementation, depending
+ * on current TYPO3 Version.
+ */
+interface TypoScriptServiceInterface
+{
+    public function convertPlainArrayToTypoScriptArray(array $plainArray);
+}

Classes/Configuration/ConfigurationContainer.php

@@ -39,7 +39,6 @@ class ConfigurationContainer implements ConfigurationContainerInterface
     /**
      * Inject settings via ConfigurationManager.
      *
-     * @param ConfigurationManagerInterface $configurationManager
      * @throws NoConfigurationException
      */
     public function injectConfigurationManager(ConfigurationManagerInterface $configurationManager)
@@ -59,7 +58,7 @@ class ConfigurationContainer implements ConfigurationContainerInterface
      * @return mixed
      * @throws InvalidArgumentException
      */
-    public function get($path)
+    public function get(string $path)
     {
         $value = ArrayUtility::getValueByPath($this->settings, $path);
 
@@ -77,7 +76,7 @@ class ConfigurationContainer implements ConfigurationContainerInterface
      * @param string $path In dot notation.
      * @return mixed
      */
-    public function getIfExists($path)
+    public function getIfExists(string $path)
     {
         return ArrayUtility::getValueByPath($this->settings, $path);
     }

Classes/Configuration/ConfigurationContainerInterface.php

@@ -20,13 +20,11 @@ namespace Codappix\SearchCore\Configuration;
  * 02110-1301, USA.
  */
 
-use TYPO3\CMS\Core\SingletonInterface as Singleton;
-
 /**
  * Container of all configurations for extension.
- * Always inject this to have a single place for configuration and parsing only once.
+ * Always inject this to have a single place for configuration.
  */
-interface ConfigurationContainerInterface extends Singleton
+interface ConfigurationContainerInterface
 {
     /**
      * Returns the option defined by section and key.
@@ -37,7 +35,7 @@ interface ConfigurationContainerInterface extends Singleton
      *
      * @throws InvalidArgumentException
      */
-    public function get($path);
+    public function get(string $path);
 
     /**
      * Same as get but will not throw an exception but return null.
@@ -45,5 +43,5 @@ interface ConfigurationContainerInterface extends Singleton
      * @param string $path In dot notation.
      * @return mixed|null
      */
-    public function getIfExists($path);
+    public function getIfExists(string $path);
 }

Classes/Connection/ConnectionInterface.php

@@ -28,62 +28,44 @@ interface ConnectionInterface
     /**
      * Will add a new document.
      *
-     * @param string $documentType
-     * @param array $document
-     *
      * @return void
      */
-    public function addDocument($documentType, array $document);
+    public function addDocument(string $documentType, array $document);
 
     /**
      * Add the given documents.
      *
-     * @param string $documentType
-     * @param array $documents
-     *
      * @return void
      */
-    public function addDocuments($documentType, array $documents);
+    public function addDocuments(string $documentType, array $documents);
 
     /**
      * Will update an existing document.
      *
      * NOTE: Batch updating is not yet supported.
      *
-     * @param string $documentType
-     * @param array $document
-     *
      * @return void
      */
-    public function updateDocument($documentType, array $document);
+    public function updateDocument(string $documentType, array $document);
 
     /**
      * Will remove an existing document.
      *
      * NOTE: Batch deleting is not yet supported.
      *
-     * @param string $documentType
-     * @param int $identifier
-     *
      * @return void
      */
-    public function deleteDocument($documentType, $identifier);
+    public function deleteDocument(string $documentType, string $identifier);
 
     /**
      * Search by given request and return result.
-     *
-     * @param SearchRequestInterface $searchRequest
-     *
-     * @return SearchResultInterface
      */
-    public function search(SearchRequestInterface $searchRequest);
+    public function search(SearchRequestInterface $searchRequest) : SearchResultInterface;
 
     /**
      * Will delete the whole index / db.
      *
-     * @param string $documentType
-     *
      * @return void
      */
-    public function deleteIndex($documentType);
+    public function deleteIndex(string $documentType);
 }

Classes/Connection/Elasticsearch.php

@@ -112,7 +112,7 @@ class Elasticsearch implements Singleton, ConnectionInterface
         $this->queryFactory = $queryFactory;
     }
 
-    public function addDocument($documentType, array $document)
+    public function addDocument(string $documentType, array $document)
     {
         $this->withType(
             $documentType,
@@ -122,7 +122,7 @@ class Elasticsearch implements Singleton, ConnectionInterface
         );
     }
 
-    public function deleteDocument($documentType, $identifier)
+    public function deleteDocument(string $documentType, string $identifier)
     {
         try {
             $this->withType(
@@ -132,11 +132,14 @@ class Elasticsearch implements Singleton, ConnectionInterface
                 }
             );
         } catch (\Elastica\Exception\NotFoundException $exception) {
-            $this->logger->debug('Tried to delete document in index, which does not exist.', [$documentType, $identifier]);
+            $this->logger->debug(
+                'Tried to delete document in index, which does not exist.',
+                [$documentType, $identifier]
+            );
         }
     }
 
-    public function updateDocument($documentType, array $document)
+    public function updateDocument(string $documentType, array $document)
     {
         $this->withType(
             $documentType,
@@ -146,7 +149,7 @@ class Elasticsearch implements Singleton, ConnectionInterface
         );
     }
 
-    public function addDocuments($documentType, array $documents)
+    public function addDocuments(string $documentType, array $documents)
     {
         $this->withType(
             $documentType,
@@ -156,7 +159,7 @@ class Elasticsearch implements Singleton, ConnectionInterface
         );
     }
 
-    public function deleteIndex($documentType)
+    public function deleteIndex(string $documentType)
     {
         $index = $this->connection->getClient()->getIndex($this->indexFactory->getIndexName());
 
@@ -173,11 +176,8 @@ class Elasticsearch implements Singleton, ConnectionInterface
 
     /**
      * Execute given callback with Elastica Type based on provided documentType
-     *
-     * @param string $documentType
-     * @param callable $callback
      */
-    protected function withType($documentType, callable $callback)
+    protected function withType(string $documentType, callable $callback)
     {
         $type = $this->getType($documentType);
         // TODO: Check whether it's to heavy to send it so often e.g. for every single document.
@@ -191,12 +191,7 @@ class Elasticsearch implements Singleton, ConnectionInterface
         $type->getIndex()->refresh();
     }
 
-    /**
-     * @param SearchRequestInterface $searchRequest
-     *
-     * @return SearchResultInterface
-     */
-    public function search(SearchRequestInterface $searchRequest)
+    public function search(SearchRequestInterface $searchRequest) : SearchResultInterface
     {
         $this->logger->debug('Search for', [$searchRequest->getSearchTerm()]);
 
@@ -207,12 +202,7 @@ class Elasticsearch implements Singleton, ConnectionInterface
         return $this->objectManager->get(SearchResult::class, $searchRequest, $search->search());
     }
 
-    /**
-     * @param string $documentType
-     *
-     * @return \Elastica\Type
-     */
-    protected function getType($documentType)
+    protected function getType(string $documentType) : \Elastica\Type
     {
         return $this->typeFactory->getType(
             $this->indexFactory->getIndex(

Classes/Connection/Elasticsearch/Connection.php

@@ -44,7 +44,7 @@ class Connection implements Singleton
 
     /**
      * @param ConfigurationContainerInterface $configuration
-     * @param \Elastica\Client $elasticaClient
+     * @param \Elastica\Client|null $elasticaClient
      */
     public function __construct(
         ConfigurationContainerInterface $configuration,
@@ -52,9 +52,8 @@ class Connection implements Singleton
     ) {
         $this->configuration = $configuration;
 
-        $this->elasticaClient = $elasticaClient;
-        if ($this->elasticaClient === null) {
-            $this->elasticaClient = new \Elastica\Client([
+        if ($elasticaClient === null) {
+            $elasticaClient = new \Elastica\Client([
                 'host' => $this->configuration->get('connections.elasticsearch.host'),
                 'port' => $this->configuration->get('connections.elasticsearch.port'),
                 // TODO: Make configurable
@@ -63,14 +62,13 @@ class Connection implements Singleton
             // TODO: Make configurable.
             // new \Elastica\Log($this->elasticaClient);
         }
+        $this->elasticaClient = $elasticaClient;
     }
 
     /**
      * Get the concrete client for internal usage!
-     *
-     * @return \Elastica\Client
      */
-    public function getClient()
+    public function getClient() : \Elastica\Client
     {
         return $this->elasticaClient;
     }

Classes/Connection/Elasticsearch/DocumentFactory.php

@@ -44,13 +44,8 @@ class DocumentFactory implements Singleton
 
     /**
      * Creates document from document.
-     *
-     * @param string $documentType
-     * @param array $document
-     *
-     * @return \Elastica\Document
      */
-    public function getDocument($documentType, array $document)
+    public function getDocument(string $documentType, array $document) : \Elastica\Document
     {
         // TODO: Use DocumentType for further configuration.
 
@@ -70,13 +65,8 @@ class DocumentFactory implements Singleton
 
     /**
      * Creates documents based on documents.
-     *
-     * @param string $documentType
-     * @param array $documents
-     *
-     * @return array
      */
-    public function getDocuments($documentType, array $documents)
+    public function getDocuments(string $documentType, array $documents) : array
     {
         foreach ($documents as &$document) {
             $document = $this->getDocument($documentType, $document);

Classes/Connection/Elasticsearch/Facet.php

@@ -22,6 +22,7 @@ namespace Codappix\SearchCore\Connection\Elasticsearch;
 
 use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
 use Codappix\SearchCore\Connection\FacetInterface;
+use Codappix\SearchCore\Connection\FacetOptionInterface;
 
 class Facet implements FacetInterface
 {
@@ -45,25 +46,26 @@ class Facet implements FacetInterface
      */
     protected $options = [];
 
-    public function __construct($name, array $aggregation, ConfigurationContainerInterface $configuration)
+    public function __construct(string $name, array $aggregation, ConfigurationContainerInterface $configuration)
     {
         $this->name = $name;
         $this->buckets = $aggregation['buckets'];
-        $this->field = $configuration->getIfExists('searching.facets.' . $this->name . '.field') ?: '';
+
+        $config = $configuration->getIfExists('searching.facets.' . $this->name) ?: [];
+        foreach ($config as $configEntry) {
+            if (isset($configEntry['field'])) {
+                $this->field = $configEntry['field'];
+                break;
+            }
+        }
     }
 
-    /**
-     * @return string
-     */
-    public function getName()
+    public function getName() : string
     {
         return $this->name;
     }
 
-    /**
-     * @return string
-     */
-    public function getField()
+    public function getField() : string
     {
         return $this->field;
     }
@@ -73,7 +75,7 @@ class Facet implements FacetInterface
      *
      * @return array<FacetOptionInterface>
      */
-    public function getOptions()
+    public function getOptions() : array
     {
         $this->initOptions();
 

Classes/Connection/Elasticsearch/FacetOption.php

@@ -29,6 +29,11 @@ class FacetOption implements FacetOptionInterface
      */
     protected $name = '';
 
+    /**
+     * @var string
+     */
+    protected $displayName = '';
+
     /**
      * @var int
      */
@@ -40,21 +45,21 @@ class FacetOption implements FacetOptionInterface
     public function __construct(array $bucket)
     {
         $this->name = $bucket['key'];
+        $this->displayName = isset($bucket['key_as_string']) ? $bucket['key_as_string'] : $this->getName();
         $this->count = $bucket['doc_count'];
     }
 
-    /**
-     * @return string
-     */
-    public function getName()
+    public function getName() : string
     {
         return $this->name;
     }
 
-    /**
-     * @return int
-     */
-    public function getCount()
+    public function getDisplayName() : string
+    {
+        return $this->displayName;
+    }
+
+    public function getCount() : int
     {
         return $this->count;
     }

Classes/Connection/Elasticsearch/IndexFactory.php

@@ -22,10 +22,8 @@ namespace Codappix\SearchCore\Connection\Elasticsearch;
 
 use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
 use Codappix\SearchCore\Configuration\InvalidArgumentException;
-use Elastica\Exception\ResponseException;
 use TYPO3\CMS\Core\SingletonInterface as Singleton;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
-use TYPO3\CMS\Extbase\Configuration\ConfigurationManagerInterface;
 
 /**
  * Factory to get indexes.
@@ -39,6 +37,21 @@ class IndexFactory implements Singleton
      */
     protected $configuration;
 
+    /**
+     * @var \TYPO3\CMS\Core\Log\Logger
+     */
+    protected $logger;
+
+    /**
+     * Inject log manager to get concrete logger from it.
+     *
+     * @param \TYPO3\CMS\Core\Log\LogManager $logManager
+     */
+    public function injectLogger(\TYPO3\CMS\Core\Log\LogManager $logManager)
+    {
+        $this->logger = $logManager->getLogger(__CLASS__);
+    }
+
     /**
      * @param ConfigurationContainerInterface $configuration
      */
@@ -59,36 +72,33 @@ class IndexFactory implements Singleton
 
     /**
      * Get an index bases on TYPO3 table name.
-     *
-     * @param Connection $connection
-     * @param string $documentType
-     *
-     * @return \Elastica\Index
      */
-    public function getIndex(Connection $connection, $documentType)
+    public function getIndex(Connection $connection, string $documentType) : \Elastica\Index
     {
         $index = $connection->getClient()->getIndex($this->getIndexName());
 
         if ($index->exists() === false) {
-            $index->create($this->getConfigurationFor($documentType));
+            $config = $this->getConfigurationFor($documentType);
+            $this->logger->debug(sprintf('Create index %s.', $documentType), [$documentType, $config]);
+            $index->create($config);
+            $this->logger->debug(sprintf('Created index %s.', $documentType), [$documentType]);
         }
 
         return $index;
     }
 
-    /**
-     * @param string $documentType
-     *
-     * @return array
-     */
-    protected function getConfigurationFor($documentType)
+    protected function getConfigurationFor(string $documentType) : array
     {
         try {
             $configuration = $this->configuration->get('indexing.' . $documentType . '.index');
 
-            if (isset($configuration['analysis']['analyzer'])) {
-                foreach ($configuration['analysis']['analyzer'] as $key => $analyzer) {
-                    $configuration['analysis']['analyzer'][$key] = $this->prepareAnalyzerConfiguration($analyzer);
+            foreach (['analyzer', 'filter'] as $optionsToExpand) {
+                if (isset($configuration['analysis'][$optionsToExpand])) {
+                    foreach ($configuration['analysis'][$optionsToExpand] as $key => $options) {
+                        $configuration['analysis'][$optionsToExpand][$key] = $this->prepareAnalyzerConfiguration(
+                            $options
+                        );
+                    }
                 }
             }
 
@@ -98,14 +108,9 @@ class IndexFactory implements Singleton
         }
     }
 
-    /**
-     * @param array $analyzer
-     *
-     * @return array
-     */
-    protected function prepareAnalyzerConfiguration(array $analyzer)
+    protected function prepareAnalyzerConfiguration(array $analyzer) : array
     {
-        $fieldsToExplode = ['char_filter', 'filter'];
+        $fieldsToExplode = ['char_filter', 'filter', 'word_list'];
 
         foreach ($fieldsToExplode as $fieldToExplode) {
             if (isset($analyzer[$fieldToExplode])) {

Classes/Connection/Elasticsearch/MappingFactory.php

@@ -44,12 +44,8 @@ class MappingFactory implements Singleton
 
     /**
      * Get an mapping based on type.
-     *
-     * @param \Elastica\Type $type
-     *
-     * @return \Elastica\Mapping
      */
-    public function getMapping(\Elastica\Type $type)
+    public function getMapping(\Elastica\Type $type) : \Elastica\Type\Mapping
     {
         $mapping = new \Elastica\Type\Mapping();
         $mapping->setType($type);
@@ -64,11 +60,7 @@ class MappingFactory implements Singleton
         return $mapping;
     }
 
-    /**
-     * @param string $identifier
-     * @return array
-     */
-    protected function getConfiguration($identifier)
+    protected function getConfiguration(string $identifier) : array
     {
         try {
             return $this->configuration->get('indexing.' . $identifier . '.mapping');

Classes/Connection/Elasticsearch/SearchResult.php

@@ -24,10 +24,14 @@ use Codappix\SearchCore\Connection\FacetInterface;
 use Codappix\SearchCore\Connection\ResultItemInterface;
 use Codappix\SearchCore\Connection\SearchRequestInterface;
 use Codappix\SearchCore\Connection\SearchResultInterface;
+use Codappix\SearchCore\Domain\Model\QueryResultInterfaceStub;
+use Codappix\SearchCore\Domain\Model\ResultItem;
 use TYPO3\CMS\Extbase\Object\ObjectManagerInterface;
 
 class SearchResult implements SearchResultInterface
 {
+    use QueryResultInterfaceStub;
+
     /**
      * @var SearchRequestInterface
      */
@@ -73,7 +77,7 @@ class SearchResult implements SearchResultInterface
     /**
      * @return array<ResultItemInterface>
      */
-    public function getResults()
+    public function getResults() : array
     {
         $this->initResults();
 
@@ -85,14 +89,14 @@ class SearchResult implements SearchResultInterface
      *
      * @return array<FacetInterface>
      */
-    public function getFacets()
+    public function getFacets() : array
     {
         $this->initFacets();
 
         return $this->facets;
     }
 
-    public function getCurrentCount()
+    public function getCurrentCount() : int
     {
         return $this->result->count();
     }
@@ -104,7 +108,7 @@ class SearchResult implements SearchResultInterface
         }
 
         foreach ($this->result->getResults() as $result) {
-            $this->results[] = new ResultItem($result);
+            $this->results[] = new ResultItem($result->getData(), $result->getParam('_type'));
         }
     }
 
@@ -153,41 +157,8 @@ class SearchResult implements SearchResultInterface
         $this->position = 0;
     }
 
-    // Extbase QueryResultInterface - Implemented to support Pagination of Fluid.
-
     public function getQuery()
     {
         return $this->searchRequest;
     }
-
-    public function getFirst()
-    {
-        throw new \BadMethodCallException('Method is not implemented yet.', 1502195121);
-    }
-
-    public function toArray()
-    {
-        throw new \BadMethodCallException('Method is not implemented yet.', 1502195135);
-    }
-
-    public function offsetExists($offset)
-    {
-        // Return false to allow Fluid to use appropriate getter methods.
-        return false;
-    }
-
-    public function offsetGet($offset)
-    {
-        throw new \BadMethodCallException('Use getter to fetch properties.', 1502196933);
-    }
-
-    public function offsetSet($offset, $value)
-    {
-        throw new \BadMethodCallException('You are not allowed to modify the result.', 1502196934);
-    }
-
-    public function offsetUnset($offset)
-    {
-        throw new \BadMethodCallException('You are not allowed to modify the result.', 1502196936);
-    }
 }

Classes/Connection/Elasticsearch/TypeFactory.php

@@ -21,7 +21,6 @@ namespace Codappix\SearchCore\Connection\Elasticsearch;
  */
 
 use TYPO3\CMS\Core\SingletonInterface as Singleton;
-use TYPO3\CMS\Extbase\Configuration\ConfigurationManagerInterface;
 
 /**
  * Factory to get indexes.
@@ -32,13 +31,8 @@ class TypeFactory implements Singleton
 {
     /**
      * Get an index bases on TYPO3 table name.
-     *
-     * @param \Elastica\Index $index
-     * @param string $documentType
-     *
-     * @return \Elastica\Type
      */
-    public function getType(\Elastica\Index $index, $documentType)
+    public function getType(\Elastica\Index $index, string $documentType) : \Elastica\Type
     {
         return $index->getType($documentType);
     }

Classes/Connection/FacetInterface.php

@@ -25,15 +25,12 @@ namespace Codappix\SearchCore\Connection;
  */
 interface FacetInterface
 {
-    /**
-     * @return string
-     */
-    public function getName();
+    public function getName() : string;
 
     /**
      * Returns all possible options for this facet.
      *
      * @return array<FacetOptionInterface>
      */
-    public function getOptions();
+    public function getOptions() : array;
 }

Classes/Connection/FacetOptionInterface.php

@@ -28,15 +28,17 @@ interface FacetOptionInterface
     /**
      * Returns the name of this option. Equivalent
      * to value used for filtering.
-     *
-     * @return string
      */
-    public function getName();
+    public function getName() : string;
+
+    /**
+     * If a pre-rendered name is provided, this will be returned.
+     * Otherwise it's the same as getName().
+     */
+    public function getDisplayName() : string;
 
     /**
      * Returns the number of found results for this option.
-     *
-     * @return int
      */
-    public function getCount();
+    public function getCount() : int;
 }

Classes/Connection/FacetRequestInterface.php

@@ -28,15 +28,11 @@ interface FacetRequestInterface
     /**
      * The identifier of the facet, used as key in arrays and to get the facet
      * from search request, etc.
-     *
-     * @return string
      */
-    public function getIdentifier();
+    public function getIdentifier() : string;
 
     /**
-     * The field to use for facet building.
-     *
-     * @return string
+     * The config to use for facet building.
      */
-    public function getField();
+    public function getConfig() : array;
 }

Classes/Connection/ResultItemInterface.php

@@ -25,5 +25,20 @@ namespace Codappix\SearchCore\Connection;
  */
 interface ResultItemInterface extends \ArrayAccess
 {
+    /**
+     * Returns every information as array.
+     *
+     * Provide key/column/field => data.
+     *
+     * Used e.g. for dataprocessing.
+     */
+    public function getPlainData() : array;
 
+    /**
+     * Returns the type of the item.
+     *
+     * That should make it easier to differentiate if multiple
+     * types are returned for one query.
+     */
+    public function getType() : string;
 }

Classes/Connection/SearchRequestInterface.php

@@ -20,24 +20,47 @@ namespace Codappix\SearchCore\Connection;
  * 02110-1301, USA.
  */
 
+use Codappix\SearchCore\Connection\ConnectionInterface;
+use Codappix\SearchCore\Connection\FacetRequestInterface;
+use Codappix\SearchCore\Domain\Search\SearchService;
 use TYPO3\CMS\Extbase\Persistence\QueryInterface;
 
 interface SearchRequestInterface extends QueryInterface
 {
     /**
      * Returns the actual string the user searched for.
+     */
+    public function getSearchTerm() : string;
+
+    public function hasFilter() : bool;
+
+    public function getFilter() : array;
+
+    public function setFilter(array $filter);
+
+    /**
+     * @return void
+     */
+    public function addFacet(FacetRequestInterface $facet);
+
+    /**
+     * @return array<FacetRequestInterface>
+     */
+    public function getFacets() : array;
+
+    /**
+     * Workaround for paginate widget support which will
+     * use the request to build another search.
      *
-     * @return string
+     * @return void
      */
-    public function getSearchTerm();
+    public function setConnection(ConnectionInterface $connection);
 
     /**
-     * @return bool
+     * Workaround for paginate widget support which will
+     * use the request to build another search.
+     *
+     * @return void
      */
-    public function hasFilter();
-
-    /**
-     * @return array
-     */
-    public function getFilter();
+    public function setSearchService(SearchService $searchService);
 }

Classes/Connection/SearchResultInterface.php

@@ -30,19 +30,17 @@ interface SearchResultInterface extends \Iterator, \Countable, QueryResultInterf
     /**
      * @return array<ResultItemInterface>
      */
-    public function getResults();
+    public function getResults() : array;
 
     /**
      * Return all facets, if any.
      *
      * @return array<FacetInterface>
      */
-    public function getFacets();
+    public function getFacets() : array;
 
     /**
      * Returns the number of results in current result
-     *
-     * @return int
      */
-    public function getCurrentCount();
+    public function getCurrentCount() : int;
 }

Classes/DataProcessing/ContentObjectDataProcessorAdapterProcessor.php

@@ -0,0 +1,59 @@
+<?php
+namespace Codappix\SearchCore\DataProcessing;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\SearchCore\Compatibility\TypoScriptServiceInterface;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Frontend\ContentObject\ContentObjectRenderer;
+
+/**
+ * Executes an existing TYPO3 DataProcessor on the given data.
+ */
+class ContentObjectDataProcessorAdapterProcessor implements ProcessorInterface
+{
+    /**
+     * @var TypoScriptServiceInterface
+     */
+    protected $typoScriptService;
+
+    public function __construct(TypoScriptServiceInterface $typoScriptService)
+    {
+        $this->typoScriptService = $typoScriptService;
+    }
+
+    public function processData(array $data, array $configuration) : array
+    {
+        $dataProcessor = GeneralUtility::makeInstance($configuration['_dataProcessor']);
+        $contentObjectRenderer = GeneralUtility::makeInstance(ContentObjectRenderer::class);
+
+        $contentObjectRenderer->data = $data;
+        if (isset($configuration['_table'])) {
+            $contentObjectRenderer->start($data, $configuration['_table']);
+        }
+
+        return $dataProcessor->process(
+            $contentObjectRenderer,
+            [],
+            $this->typoScriptService->convertPlainArrayToTypoScriptArray($configuration),
+            $data
+        );
+    }
+}

Classes/DataProcessing/CopyToProcessor.php

@@ -25,13 +25,23 @@ namespace Codappix\SearchCore\DataProcessing;
  */
 class CopyToProcessor implements ProcessorInterface
 {
-    public function processRecord(array $record, array $configuration) : array
+    public function processData(array $record, array $configuration) : array
     {
-        $all = [];
+        $target = [];
 
-        $this->addArray($all, $record);
-        $all = array_filter($all);
-        $record[$configuration['to']] = implode(PHP_EOL, $all);
+        $from = $record;
+        if (isset($configuration['from'])) {
+            $from = $record[$configuration['from']];
+        }
+
+        if (is_array($from)) {
+            $this->addArray($target, $from);
+        } else {
+            $target[] = (string) $from;
+        }
+
+        $target = array_filter($target);
+        $record[$configuration['to']] = implode(PHP_EOL, $target);
 
         return $record;
     }

Classes/DataProcessing/GeoPointProcessor.php

@@ -25,9 +25,9 @@ namespace Codappix\SearchCore\DataProcessing;
  */
 class GeoPointProcessor implements ProcessorInterface
 {
-    public function processRecord(array $record, array $configuration) : array
+    public function processData(array $record, array $configuration) : array
     {
-        if (! $this->canApply($record, $configuration)) {
+        if (! $this->isApplyable($record, $configuration)) {
             return $record;
         }
 
@@ -39,7 +39,7 @@ class GeoPointProcessor implements ProcessorInterface
         return $record;
     }
 
-    protected function canApply(array $record, array $configuration) : bool
+    protected function isApplyable(array $record, array $configuration) : bool
     {
         if (!isset($record[$configuration['lat']])
             || !is_numeric($record[$configuration['lat']])

Classes/DataProcessing/ProcessorInterface.php

@@ -21,14 +21,13 @@ namespace Codappix\SearchCore\DataProcessing;
  */
 
 /**
- * All DataProcessing Processors should implement this interface, otherwise they
- * will not be executed.
+ * All DataProcessing Processors should implement this interface.
  */
 interface ProcessorInterface
 {
     /**
-     * Processes the given record.
+     * Processes the given data.
      * Also retrieves the configuration for this processor instance.
      */
-    public function processRecord(array $record, array $configuration) : array;
+    public function processData(array $record, array $configuration) : array;
 }

Classes/DataProcessing/RemoveProcessor.php

@@ -27,7 +27,7 @@ use TYPO3\CMS\Core\Utility\GeneralUtility;
  */
 class RemoveProcessor implements ProcessorInterface
 {
-    public function processRecord(array $record, array $configuration) : array
+    public function processData(array $record, array $configuration) : array
     {
         if (!isset($configuration['fields'])) {
             return $record;

Classes/DataProcessing/Service.php

@@ -0,0 +1,60 @@
+<?php
+namespace Codappix\SearchCore\DataProcessing;
+
+/*
+ * Copyright (C) 2018  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use TYPO3\CMS\Extbase\Object\ObjectManagerInterface;
+
+/**
+ * Eases work with data processing.
+ */
+class Service
+{
+    /**
+     * @var ObjectManagerInterface
+     */
+    protected $objectManager;
+
+    public function __construct(ObjectManagerInterface $objectManager)
+    {
+        $this->objectManager = $objectManager;
+    }
+
+    /**
+     * Executes the dataprocessor depending on configuration and returns the result.
+     *
+     * @param array|string $configuration Either the full configuration or only the class name.
+     */
+    public function executeDataProcessor($configuration, array $data, string $recordType = '') : array
+    {
+        if (is_string($configuration)) {
+            $configuration = [
+                '_typoScriptNodeValue' => $configuration,
+            ];
+        }
+
+        if (!isset($configuration['_table']) && $recordType !== '') {
+            $configuration['_table'] = $recordType;
+        }
+
+        return $this->objectManager->get($configuration['_typoScriptNodeValue'])
+            ->processData($data, $configuration);
+    }
+}

Classes/DataProcessing/TcaRelationResolvingProcessor.php

@@ -0,0 +1,101 @@
+<?php
+namespace Codappix\SearchCore\DataProcessing;
+
+/*
+ * Copyright (C) 2018  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\SearchCore\Domain\Index\TcaIndexer\RelationResolver;
+use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Extbase\Object\ObjectManagerInterface;
+
+/**
+ * Resolves relations from TCA using RelationResolver.
+ */
+class TcaRelationResolvingProcessor implements ProcessorInterface
+{
+    /**
+     * @var ObjectManagerInterface
+     */
+    protected $objectManager;
+
+    /**
+     * @var RelationResolver
+     */
+    protected $relationResolver;
+
+    public function __construct(
+        ObjectManagerInterface $objectManager,
+        RelationResolver $relationResolver
+    ) {
+        $this->objectManager = $objectManager;
+        $this->relationResolver = $relationResolver;
+    }
+
+    /**
+     * @throws \InvalidArgumentException If _table is not configured.
+     */
+    public function processData(array $record, array $configuration) : array
+    {
+        $this->initializeConfiguration($configuration);
+
+        $tcaTableService = $this->objectManager->get(
+            TcaTableServiceInterface::class,
+            $configuration['_table']
+        );
+
+        $processedRecord = $this->relationResolver->resolveRelationsForRecord(
+            $tcaTableService,
+            $this->getRecordToProcess($record, $configuration)
+        );
+
+        return array_merge($record, $processedRecord);
+    }
+
+    /**
+     * @throws \InvalidArgumentException If _table is not configured.
+     */
+    protected function initializeConfiguration(array &$configuration)
+    {
+        if (!isset($configuration['_table'])) {
+            throw new \InvalidArgumentException('The configuration "_table" is mandantory.', 1524552631);
+        }
+
+        if (!isset($configuration['excludeFields'])) {
+            $configuration['excludeFields'] = '';
+        }
+
+        $configuration['excludeFields'] = GeneralUtility::trimExplode(',', $configuration['excludeFields'], true);
+    }
+
+    protected function getRecordToProcess(array $record, array $configuration) : array
+    {
+        if ($configuration['excludeFields'] === []) {
+            return $record;
+        }
+
+        $newRecord = [];
+        $keysToUse = array_diff(array_keys($record), $configuration['excludeFields']);
+        foreach ($keysToUse as $keyToUse) {
+            $newRecord[$keyToUse] = $record[$keyToUse];
+        }
+
+        return $newRecord;
+    }
+}

Classes/Domain/Index/AbstractIndexer.php

@@ -23,7 +23,6 @@ namespace Codappix\SearchCore\Domain\Index;
 use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
 use Codappix\SearchCore\Configuration\InvalidArgumentException;
 use Codappix\SearchCore\Connection\ConnectionInterface;
-use Codappix\SearchCore\DataProcessing\ProcessorInterface;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 
 abstract class AbstractIndexer implements IndexerInterface
@@ -43,6 +42,12 @@ abstract class AbstractIndexer implements IndexerInterface
      */
     protected $identifier = '';
 
+    /**
+     * @var \Codappix\SearchCore\DataProcessing\Service
+     * @inject
+     */
+    protected $dataProcessorService;
+
     /**
      * @var \TYPO3\CMS\Core\Log\Logger
      */
@@ -58,15 +63,11 @@ abstract class AbstractIndexer implements IndexerInterface
         $this->logger = $logManager->getLogger(__CLASS__);
     }
 
-    public function setIdentifier($identifier)
+    public function setIdentifier(string $identifier)
     {
         $this->identifier = $identifier;
     }
 
-    /**
-     * @param ConnectionInterface $connection
-     * @param ConfigurationContainerInterface $configuration
-     */
     public function __construct(ConnectionInterface $connection, ConfigurationContainerInterface $configuration)
     {
         $this->connection = $connection;
@@ -91,11 +92,11 @@ abstract class AbstractIndexer implements IndexerInterface
         $this->logger->info('Finish indexing');
     }
 
-    public function indexDocument($identifier)
+    public function indexDocument(string $identifier)
     {
         $this->logger->info('Start indexing single record.', [$identifier]);
         try {
-            $record = $this->getRecord($identifier);
+            $record = $this->getRecord((int) $identifier);
             $this->prepareRecord($record);
 
             $this->connection->addDocument($this->getDocumentName(), $record);
@@ -113,10 +114,7 @@ abstract class AbstractIndexer implements IndexerInterface
         $this->logger->info('Finish deletion.');
     }
 
-    /**
-     * @return \Generator
-     */
-    protected function getRecordGenerator()
+    protected function getRecordGenerator() : \Generator
     {
         $offset = 0;
         $limit = $this->getLimit();
@@ -127,24 +125,11 @@ abstract class AbstractIndexer implements IndexerInterface
         }
     }
 
-    /**
-     * @param array &$record
-     */
     protected function prepareRecord(array &$record)
     {
         try {
             foreach ($this->configuration->get('indexing.' . $this->identifier . '.dataProcessing') as $configuration) {
-                $className = '';
-                if (is_string($configuration)) {
-                    $className = $configuration;
-                    $configuration = [];
-                } else {
-                    $className = $configuration['_typoScriptNodeValue'];
-                }
-                $dataProcessor = GeneralUtility::makeInstance($className);
-                if ($dataProcessor instanceof ProcessorInterface) {
-                    $record = $dataProcessor->processRecord($record, $configuration);
-                }
+                $record = $this->dataProcessorService->executeDataProcessor($configuration, $record, $this->identifier);
             }
         } catch (InvalidArgumentException $e) {
             // Nothing to do.
@@ -153,9 +138,6 @@ abstract class AbstractIndexer implements IndexerInterface
         $this->handleAbstract($record);
     }
 
-    /**
-     * @param array &$record
-     */
     protected function handleAbstract(array &$record)
     {
         $record['search_abstract'] = '';
@@ -165,7 +147,7 @@ abstract class AbstractIndexer implements IndexerInterface
                 ',',
                 $this->configuration->get('indexing.' . $this->identifier . '.abstractFields')
             );
-            if (!$fieldsToUse) {
+            if ($fieldsToUse === []) {
                 return;
             }
             foreach ($fieldsToUse as $fieldToUse) {
@@ -181,31 +163,22 @@ abstract class AbstractIndexer implements IndexerInterface
 
     /**
      * Returns the limit to use to fetch records.
-     *
-     * @return int
      */
-    protected function getLimit()
+    protected function getLimit() : int
     {
         // TODO: Make configurable.
         return 50;
     }
 
     /**
-     * @param int $offset
-     * @param int $limit
      * @return array|null
      */
-    abstract protected function getRecords($offset, $limit);
+    abstract protected function getRecords(int $offset, int $limit);
 
     /**
-     * @param int $identifier
-     * @return array
      * @throws NoRecordFoundException If record could not be found.
      */
-    abstract protected function getRecord($identifier);
+    abstract protected function getRecord(int $identifier) : array;
 
-    /**
-     * @return string
-     */
-    abstract protected function getDocumentName();
+    abstract protected function getDocumentName() : string;
 }

Classes/Domain/Index/IndexerFactory.php

@@ -23,8 +23,7 @@ namespace Codappix\SearchCore\Domain\Index;
 use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
 use Codappix\SearchCore\Configuration\InvalidArgumentException;
 use Codappix\SearchCore\Domain\Index\IndexerInterface;
-use Codappix\SearchCore\Domain\Index\TcaIndexer;
-use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService;
+use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface;
 use TYPO3\CMS\Core\SingletonInterface as Singleton;
 use TYPO3\CMS\Extbase\Object\ObjectManagerInterface;
 
@@ -56,12 +55,9 @@ class IndexerFactory implements Singleton
     }
 
     /**
-     * @param string $identifier
-     *
-     * @return IndexerInterface
      * @throws NoMatchingIndexer
      */
-    public function getIndexer($identifier)
+    public function getIndexer(string $identifier) : IndexerInterface
     {
         try {
             return $this->buildIndexer($this->configuration->get('indexing.' . $identifier . '.indexer'), $identifier);
@@ -75,13 +71,9 @@ class IndexerFactory implements Singleton
     }
 
     /**
-     * @param string $indexerClass
-     * @param string $identifier
-     *
-     * @return IndexerInterface
      * @throws NoMatchingIndexer
      */
-    protected function buildIndexer($indexerClass, $identifier)
+    protected function buildIndexer(string $indexerClass, string $identifier) : IndexerInterface
     {
         $indexer = null;
         if (is_subclass_of($indexerClass, TcaIndexer\PagesIndexer::class)
@@ -89,13 +81,13 @@ class IndexerFactory implements Singleton
         ) {
             $indexer = $this->objectManager->get(
                 $indexerClass,
-                $this->objectManager->get(TcaTableService::class, $identifier),
-                $this->objectManager->get(TcaTableService::class, 'tt_content')
+                $this->objectManager->get(TcaTableServiceInterface::class, $identifier),
+                $this->objectManager->get(TcaTableServiceInterface::class, 'tt_content')
             );
         } elseif (is_subclass_of($indexerClass, TcaIndexer::class) || $indexerClass === TcaIndexer::class) {
             $indexer = $this->objectManager->get(
                 $indexerClass,
-                $this->objectManager->get(TcaTableService::class, $identifier)
+                $this->objectManager->get(TcaTableServiceInterface::class, $identifier)
             );
         } elseif (class_exists($indexerClass) && in_array(IndexerInterface::class, class_implements($indexerClass))) {
             $indexer = $this->objectManager->get($indexerClass);

Classes/Domain/Index/IndexerInterface.php

@@ -35,20 +35,16 @@ interface IndexerInterface
     /**
      * Fetches a single document and pushes it to the connection.
      *
-     * @param string $identifier
-     *
      * @return void
      */
-    public function indexDocument($identifier);
+    public function indexDocument(string $identifier);
 
     /**
      * Recieves the identifier of the indexer itself.
      *
-     * @param string $identifier
-     *
      * @return void
      */
-    public function setIdentifier($identifier);
+    public function setIdentifier(string $identifier);
 
     /**
      * Delete the whole index.

Classes/Domain/Index/TcaIndexer.php

@@ -22,9 +22,7 @@ namespace Codappix\SearchCore\Domain\Index;
 
 use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
 use Codappix\SearchCore\Connection\ConnectionInterface;
-use TYPO3\CMS\Core\Database\ConnectionPool;
-use TYPO3\CMS\Core\Database\Query\QueryBuilder;
-use TYPO3\CMS\Core\Utility\GeneralUtility;
+use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface;
 
 /**
  * Will index the given table using configuration from TCA.
@@ -32,39 +30,31 @@ use TYPO3\CMS\Core\Utility\GeneralUtility;
 class TcaIndexer extends AbstractIndexer
 {
     /**
-     * @var TcaIndexer\TcaTableService
+     * @var TcaTableServiceInterface
      */
     protected $tcaTableService;
 
     /**
-     * @param TcaIndexer\TcaTableService $tcaTableService
+     * @param TcaTableServiceInterface $tcaTableService
      * @param ConnectionInterface $connection
      * @param ConfigurationContainerInterface $configuration
      */
     public function __construct(
-        TcaIndexer\TcaTableService $tcaTableService,
+        TcaTableServiceInterface $tcaTableService,
         ConnectionInterface $connection,
         ConfigurationContainerInterface $configuration
     ) {
+        parent::__construct($connection, $configuration);
         $this->tcaTableService = $tcaTableService;
-        $this->connection = $connection;
-        $this->configuration = $configuration;
     }
 
     /**
-     * @param int $offset
-     * @param int $limit
      * @return array|null
      */
-    protected function getRecords($offset, $limit)
+    protected function getRecords(int $offset, int $limit)
     {
-        $records = $this->getQuery()
-            ->setFirstResult($offset)
-            ->setMaxResults($limit)
-            ->execute()
-            ->fetchAll();
-
-        if ($records === null) {
+        $records = $this->tcaTableService->getRecords($offset, $limit);
+        if ($records === []) {
             return null;
         }
 
@@ -77,17 +67,13 @@ class TcaIndexer extends AbstractIndexer
     }
 
     /**
-     * @param int $identifier
-     * @return array
      * @throws NoRecordFoundException If record could not be found.
      */
-    protected function getRecord($identifier)
+    protected function getRecord(int $identifier) : array
     {
-        $query = $this->getQuery();
-        $query = $query->andWhere($this->tcaTableService->getTableName() . '.uid = ' . (int) $identifier);
-        $record = $query->execute()->fetch();
+        $record = $this->tcaTableService->getRecord($identifier);
 
-        if ($record === false || $record === null) {
+        if ($record === []) {
             throw new NoRecordFoundException(
                 'Record could not be fetched from database: "' . $identifier . '". Perhaps record is not active.',
                 1484225364
@@ -98,36 +84,8 @@ class TcaIndexer extends AbstractIndexer
         return $record;
     }
 
-    /**
-     * @return string
-     */
-    protected function getDocumentName()
+    protected function getDocumentName() : string
     {
         return $this->tcaTableService->getTableName();
     }
-
-    protected function getQuery($tcaTableService = null) : QueryBuilder
-    {
-        if ($tcaTableService === null) {
-            $tcaTableService = $this->tcaTableService;
-        }
-        $queryBuilder = $this->getDatabaseConnection()->getQueryBuilderForTable($tcaTableService->getTableName());
-        $where = $tcaTableService->getWhereClause();
-        $query = $queryBuilder->select(... $tcaTableService->getFields())
-            ->from($tcaTableService->getTableClause())
-            ->where($where->getStatement())
-            ->setParameters($where->getParameters());
-
-        foreach ($tcaTableService->getJoins() as $join) {
-            $query->from($join->getTable());
-            $query->andWhere($join->getCondition());
-        }
-
-        return $query;
-    }
-
-    protected function getDatabaseConnection()
-    {
-        return GeneralUtility::makeInstance(ConnectionPool::class);
-    }
 }

Classes/Domain/Index/TcaIndexer/PagesIndexer.php

@@ -23,6 +23,8 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer;
 use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
 use Codappix\SearchCore\Connection\ConnectionInterface;
 use Codappix\SearchCore\Domain\Index\TcaIndexer;
+use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
 
 /**
  * Specific indexer for Pages, will basically add content of page.
@@ -30,33 +32,36 @@ use Codappix\SearchCore\Domain\Index\TcaIndexer;
 class PagesIndexer extends TcaIndexer
 {
     /**
-     * @var TcaTableService
+     * @var TcaTableServiceInterface
      */
     protected $contentTableService;
 
     /**
-     * @param TcaTableService $tcaTableService
-     * @param TcaTableService $tcaTableService
+     * @var \TYPO3\CMS\Core\Resource\FileRepository
+     * @inject
+     */
+    protected $fileRepository;
+
+    /**
+     * @param TcaTableServiceInterface $tcaTableService
+     * @param TcaTableServiceInterface $contentTableService
      * @param ConnectionInterface $connection
      * @param ConfigurationContainerInterface $configuration
      */
     public function __construct(
-        TcaTableService $tcaTableService,
-        TcaTableService $contentTableService,
+        TcaTableServiceInterface $tcaTableService,
+        TcaTableServiceInterface $contentTableService,
         ConnectionInterface $connection,
         ConfigurationContainerInterface $configuration
     ) {
-        $this->tcaTableService = $tcaTableService;
+        parent::__construct($tcaTableService, $connection, $configuration);
         $this->contentTableService = $contentTableService;
-        $this->connection = $connection;
-        $this->configuration = $configuration;
     }
 
-    /**
-     * @param array &$record
-     */
     protected function prepareRecord(array &$record)
     {
+        parent::prepareRecord($record);
+
         $possibleTitleFields = ['nav_title', 'tx_tqseo_pagetitle_rel', 'title'];
         foreach ($possibleTitleFields as $searchTitleField) {
             if (isset($record[$searchTitleField]) && trim($record[$searchTitleField])) {
@@ -65,32 +70,96 @@ class PagesIndexer extends TcaIndexer
             }
         }
 
-        $record['content'] = $this->fetchContentForPage($record['uid']);
-        parent::prepareRecord($record);
+        $record['media'] = $this->fetchMediaForPage($record['uid']);
+        $content = $this->fetchContentForPage($record['uid']);
+        if ($content !== []) {
+            $record['content'] = $content['content'];
+            $record['media'] = array_values(array_unique(array_merge($record['media'], $content['images'])));
+        }
     }
 
-    /**
-     * @param int $uid
-     * @return string
-     */
-    protected function fetchContentForPage($uid)
+    protected function fetchContentForPage(int $uid) : array
     {
-        $contentElements = $this->getQuery($this->contentTableService)->execute()->fetchAll();
+        if ($this->contentTableService instanceof TcaTableService) {
+            $queryBuilder = $this->contentTableService->getQuery();
+            $queryBuilder->andWhere(
+                $queryBuilder->expr()->eq(
+                    $this->contentTableService->getTableName() . '.pid',
+                    $queryBuilder->createNamedParameter($uid, \PDO::PARAM_INT)
+                )
+            );
+            $contentElements = $queryBuilder->execute()->fetchAll();
+        } else {
+            $contentElements = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows(
+                $this->contentTableService->getFields(),
+                $this->contentTableService->getTableClause(),
+                $this->contentTableService->getWhereClause() .
+                sprintf(' AND %s.pid = %u', $this->contentTableService->getTableName(), $uid)
+            );
+        }
 
         if ($contentElements === null) {
             $this->logger->debug('No content for page ' . $uid);
-            return '';
+            return [];
         }
 
         $this->logger->debug('Fetched content for page ' . $uid);
+        $images = [];
         $content = [];
         foreach ($contentElements as $contentElement) {
-            $content[] = $contentElement['bodytext'];
+            $images = array_merge(
+                $images,
+                $this->getContentElementImages($contentElement['uid'])
+            );
+            $content[] = $this->getContentFromContentElement($contentElement);
         }
 
-        // Remove Tags.
-        // Interpret escaped new lines and special chars.
-        // Trim, e.g. trailing or leading new lines.
-        return trim(stripcslashes(strip_tags(implode(' ', $content))));
+        return [
+            // Remove Tags.
+            // Interpret escaped new lines and special chars.
+            // Trim, e.g. trailing or leading new lines.
+            'content' => trim(stripcslashes(strip_tags(implode(' ', $content)))),
+            'images' => $images,
+        ];
+    }
+
+    protected function getContentElementImages(int $uidOfContentElement) : array
+    {
+        return $this->fetchSysFileReferenceUids($uidOfContentElement, 'tt_content', 'image');
+    }
+
+    protected function fetchMediaForPage(int $uid) : array
+    {
+        return $this->fetchSysFileReferenceUids($uid, 'pages', 'media');
+    }
+
+    protected function fetchSysFileReferenceUids(int $uid, string $tablename, string $fieldname) : array
+    {
+        $imageRelationUids = [];
+        $imageRelations = $this->fileRepository->findByRelation($tablename, $fieldname, $uid);
+
+        foreach ($imageRelations as $relation) {
+            $imageRelationUids[] = $relation->getUid();
+        }
+
+        return $imageRelationUids;
+    }
+
+    protected function getContentFromContentElement(array $contentElement) : string
+    {
+        $content = '';
+
+        $fieldsWithContent = GeneralUtility::trimExplode(
+            ',',
+            $this->configuration->get('indexing.' . $this->identifier . '.contentFields'),
+            true
+        );
+        foreach ($fieldsWithContent as $fieldWithContent) {
+            if (isset($contentElement[$fieldWithContent]) && trim($contentElement[$fieldWithContent])) {
+                $content .= trim($contentElement[$fieldWithContent]) . ' ';
+            }
+        }
+
+        return trim($content);
     }
 }

Classes/Domain/Index/TcaIndexer/RelationResolver.php

@@ -33,11 +33,11 @@ use TYPO3\CMS\Core\Utility\GeneralUtility;
  */
 class RelationResolver implements Singleton
 {
-    public function resolveRelationsForRecord(TcaTableService $service, array &$record) : void
+    public function resolveRelationsForRecord(TcaTableServiceInterface $service, array $record) : array
     {
         foreach (array_keys($record) as $column) {
-            // TODO: Define / configure fields to exclude?!
-            if ($column === 'pid') {
+            if (in_array($column, ['pid', $service->getLanguageUidColumn()])) {
+                $record[$column] = (int) $record[$column];
                 continue;
             }
 
@@ -61,6 +61,8 @@ class RelationResolver implements Singleton
                 continue;
             }
         }
+
+        return $record;
     }
 
     protected function resolveValue($value, array $tcaColumn)
@@ -82,7 +84,7 @@ class RelationResolver implements Singleton
     protected function isRelation(array &$config) : bool
     {
         return isset($config['foreign_table'])
-            || (isset($config['renderType']) && $config['renderType'] !== 'selectSingle')
+            || (isset($config['renderType']) && !in_array($config['renderType'], ['selectSingle', 'inputDateTime']))
             || (isset($config['internal_type']) && strtolower($config['internal_type']) === 'db')
             ;
     }
@@ -97,7 +99,7 @@ class RelationResolver implements Singleton
         return array_map('trim', explode(',', $value));
     }
 
-    protected function getUtilityForMode(): string
+    protected function getUtilityForMode() : string
     {
         if (TYPO3_MODE === 'BE') {
             return BackendUtility::class;

Classes/Domain/Index/TcaIndexer/TcaTableService.php

@@ -21,11 +21,13 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer;
  */
 
 use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
-use Codappix\SearchCore\Configuration\InvalidArgumentException as InvalidConfigurationArgumentException;
 use Codappix\SearchCore\Database\Doctrine\Join;
 use Codappix\SearchCore\Database\Doctrine\Where;
 use Codappix\SearchCore\Domain\Index\IndexingException;
+use Codappix\SearchCore\Domain\Index\TcaIndexer\InvalidArgumentException;
 use TYPO3\CMS\Backend\Utility\BackendUtility;
+use TYPO3\CMS\Core\Database\ConnectionPool;
+use TYPO3\CMS\Core\Database\Query\QueryBuilder;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 use TYPO3\CMS\Core\Utility\RootlineUtility;
 use TYPO3\CMS\Extbase\Object\ObjectManagerInterface;
@@ -33,7 +35,7 @@ use TYPO3\CMS\Extbase\Object\ObjectManagerInterface;
 /**
  * Encapsulate logik related to TCA configuration.
  */
-class TcaTableService
+class TcaTableService implements TcaTableServiceInterface
 {
     /**
      * TCA for current table.
@@ -52,11 +54,6 @@ class TcaTableService
      */
     protected $configuration;
 
-    /**
-     * @var RelationResolver
-     */
-    protected $relationResolver;
-
     /**
      * @var \TYPO3\CMS\Core\Log\Logger
      */
@@ -91,7 +88,6 @@ class TcaTableService
      */
     public function __construct(
         $tableName,
-        RelationResolver $relationResolver,
         ConfigurationContainerInterface $configuration
     ) {
         if (!isset($GLOBALS['TCA'][$tableName])) {
@@ -104,32 +100,39 @@ class TcaTableService
         $this->tableName = $tableName;
         $this->tca = &$GLOBALS['TCA'][$this->tableName];
         $this->configuration = $configuration;
-        $this->relationResolver = $relationResolver;
     }
 
-    /**
-     * @return string
-     */
     public function getTableName() : string
     {
         return $this->tableName;
     }
 
-    /**
-     * @return string
-     */
     public function getTableClause() : string
     {
         return $this->tableName;
     }
 
-    /**
-     * Filter the given records by root line blacklist settings.
-     *
-     * @param array &$records
-     * @return void
-     */
-    public function filterRecordsByRootLineBlacklist(array &$records) : void
+    public function getRecords(int $offset, int $limit) : array
+    {
+        $records = $this->getQuery()
+            ->setFirstResult($offset)
+            ->setMaxResults($limit)
+            ->execute()
+            ->fetchAll();
+
+        return $records ?: [];
+    }
+
+    public function getRecord(int $identifier) : array
+    {
+        $query = $this->getQuery();
+        $query = $query->andWhere($this->getTableName() . '.uid = ' . $identifier);
+        $record = $query->execute()->fetch();
+
+        return $record ?: [];
+    }
+
+    public function filterRecordsByRootLineBlacklist(array &$records)
     {
         $records = array_filter(
             $records,
@@ -139,13 +142,8 @@ class TcaTableService
         );
     }
 
-    /**
-     * @param array &$record
-     */
-    public function prepareRecord(array &$record) : void
+    public function prepareRecord(array &$record)
     {
-        $this->relationResolver->resolveRelationsForRecord($this, $record);
-
         if (isset($record['uid']) && !isset($record['search_identifier'])) {
             $record['search_identifier'] = $record['uid'];
         }
@@ -154,18 +152,20 @@ class TcaTableService
         }
     }
 
-    public function getWhereClause() : Where
+    protected function getWhereClause() : Where
     {
         $parameters = [];
         $whereClause = $this->getSystemWhereClause();
 
-        $userDefinedWhere = $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.additionalWhereClause');
+        $userDefinedWhere = $this->configuration->getIfExists(
+            'indexing.' . $this->getTableName() . '.additionalWhereClause'
+        );
         if (is_string($userDefinedWhere)) {
             $whereClause .= ' AND ' . $userDefinedWhere;
         }
 
-        if ($this->isBlacklistedRootLineConfigured()) {
-            $parameters[':blacklistedRootLine'] = $this->getBlacklistedRootLine();
+        if ($this->isBlackListedRootLineConfigured()) {
+            $parameters[':blacklistedRootLine'] = implode(',', $this->getBlackListedRootLine());
             $whereClause .= ' AND pages.uid NOT IN (:blacklistedRootLine)'
                 . ' AND pages.pid NOT IN (:blacklistedRootLine)';
         }
@@ -174,7 +174,7 @@ class TcaTableService
         return new Where($whereClause, $parameters);
     }
 
-    public function getFields() : array
+    protected function getFields() : array
     {
         $fields = array_merge(
             ['uid','pid'],
@@ -197,7 +197,7 @@ class TcaTableService
         return $fields;
     }
 
-    public function getJoins() : array
+    protected function getJoins() : array
     {
         if ($this->tableName === 'pages') {
             return [];
@@ -212,7 +212,7 @@ class TcaTableService
      * Generate SQL for TYPO3 as a system, to make sure only available records
      * are fetched.
      */
-    public function getSystemWhereClause() : string
+    protected function getSystemWhereClause() : string
     {
         $whereClause = '1=1'
             . BackendUtility::BEenableFields($this->tableName)
@@ -229,11 +229,7 @@ class TcaTableService
         return $whereClause;
     }
 
-    /**
-     * @param string
-     * @return bool
-     */
-    protected function isSystemField($columnName) : bool
+    protected function isSystemField(string $columnName) : bool
     {
         $systemFields = [
             // Versioning fields,
@@ -245,7 +241,6 @@ class TcaTableService
             $this->tca['ctrl']['cruser_id'],
             $this->tca['ctrl']['fe_cruser_id'],
             $this->tca['ctrl']['fe_crgroup_id'],
-            $this->tca['ctrl']['languageField'],
             $this->tca['ctrl']['origUid'],
         ];
 
@@ -265,11 +260,9 @@ class TcaTableService
     }
 
     /**
-     * @param string $columnName
-     * @return array
      * @throws InvalidArgumentException
      */
-    public function getColumnConfig($columnName) : array
+    public function getColumnConfig(string $columnName) : array
     {
         if (!isset($this->tca['columns'][$columnName])) {
             throw new InvalidArgumentException(
@@ -281,6 +274,15 @@ class TcaTableService
         return $this->tca['columns'][$columnName]['config'];
     }
 
+    public function getLanguageUidColumn() : string
+    {
+        if (!isset($this->tca['ctrl']['languageField'])) {
+            return '';
+        }
+
+        return $this->tca['ctrl']['languageField'];
+    }
+
     /**
      * Checks whether the given record was blacklisted by root line.
      * This can be configured by typoscript as whole root lines can be black listed.
@@ -288,9 +290,6 @@ class TcaTableService
      * Also further TYPO3 mechanics are taken into account. Does a valid root
      * line exist, is page inside a recycler, is inherited start- endtime
      * excluded, etc.
-     *
-     * @param array &$record
-     * @return bool
      */
     protected function isRecordBlacklistedByRootline(array &$record) : bool
     {
@@ -346,8 +345,6 @@ class TcaTableService
 
     /**
      * Checks whether any page uids are black listed.
-     *
-     * @return bool
      */
     protected function isBlackListedRootLineConfigured() : bool
     {
@@ -361,6 +358,31 @@ class TcaTableService
      */
     protected function getBlackListedRootLine() : array
     {
-        return GeneralUtility::intExplode(',', $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist'));
+        return GeneralUtility::intExplode(
+            ',',
+            $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist')
+        );
+    }
+
+    public function getQuery() : QueryBuilder
+    {
+        $queryBuilder = $this->getDatabaseConnection()->getQueryBuilderForTable($this->getTableName());
+        $where = $this->getWhereClause();
+        $query = $queryBuilder->select(... $this->getFields())
+            ->from($this->getTableClause())
+            ->where($where->getStatement())
+            ->setParameters($where->getParameters());
+
+        foreach ($this->getJoins() as $join) {
+            $query->from($join->getTable());
+            $query->andWhere($join->getCondition());
+        }
+
+        return $query;
+    }
+
+    protected function getDatabaseConnection() : ConnectionPool
+    {
+        return GeneralUtility::makeInstance(ConnectionPool::class);
     }
 }

Classes/Domain/Index/TcaIndexer/TcaTableService76.php

@@ -0,0 +1,378 @@
+<?php
+namespace Codappix\SearchCore\Domain\Index\TcaIndexer;
+
+/*
+ * Copyright (C) 2016  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
+use Codappix\SearchCore\Domain\Index\IndexingException;
+use Codappix\SearchCore\Domain\Index\TcaIndexer\InvalidArgumentException;
+use TYPO3\CMS\Backend\Utility\BackendUtility;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Core\Utility\RootlineUtility;
+use TYPO3\CMS\Extbase\Object\ObjectManagerInterface;
+
+/**
+ * Encapsulate logik related to TCA configuration.
+ */
+class TcaTableService76 implements TcaTableServiceInterface
+{
+    /**
+     * TCA for current table.
+     * !REFERENCE! To save memory.
+     * @var array
+     */
+    protected $tca;
+
+    /**
+     * @var string
+     */
+    protected $tableName;
+
+    /**
+     * @var ConfigurationContainerInterface
+     */
+    protected $configuration;
+
+    /**
+     * @var \TYPO3\CMS\Core\Log\Logger
+     */
+    protected $logger;
+
+    /**
+     * @var ObjectManagerInterface
+     */
+    protected $objectManager;
+
+    /**
+     * Inject log manager to get concrete logger from it.
+     *
+     * @param \TYPO3\CMS\Core\Log\LogManager $logManager
+     */
+    public function injectLogger(\TYPO3\CMS\Core\Log\LogManager $logManager)
+    {
+        $this->logger = $logManager->getLogger(__CLASS__);
+    }
+
+    /**
+     * @param ObjectManagerInterface $objectManager
+     */
+    public function injectObjectManager(ObjectManagerInterface $objectManager)
+    {
+        $this->objectManager = $objectManager;
+    }
+
+    /**
+     * @param string $tableName
+     * @param ConfigurationContainerInterface $configuration
+     */
+    public function __construct(
+        $tableName,
+        ConfigurationContainerInterface $configuration
+    ) {
+        if (!isset($GLOBALS['TCA'][$tableName])) {
+            throw new IndexingException(
+                'Table "' . $tableName . '" is not configured in TCA.',
+                IndexingException::CODE_UNKOWN_TCA_TABLE
+            );
+        }
+
+        $this->tableName = $tableName;
+        $this->tca = &$GLOBALS['TCA'][$this->tableName];
+        $this->configuration = $configuration;
+    }
+
+    public function getTableName() : string
+    {
+        return $this->tableName;
+    }
+
+    public function getTableClause() : string
+    {
+        if ($this->tableName === 'pages') {
+            return $this->tableName;
+        }
+
+        return $this->tableName . ' LEFT JOIN pages on ' . $this->tableName . '.pid = pages.uid';
+    }
+
+    public function getRecords(int $offset, int $limit) : array
+    {
+        $records = $this->getConnection()->exec_SELECTgetRows(
+            $this->getFields(),
+            $this->getTableClause(),
+            $this->getWhereClause(),
+            '',
+            '',
+            (int) $offset . ',' . (int) $limit
+        );
+
+        return $records ?: [];
+    }
+
+    public function getRecord(int $identifier) : array
+    {
+        $record = $this->getConnection()->exec_SELECTgetSingleRow(
+            $this->getFields(),
+            $this->getTableClause(),
+            $this->getWhereClause()
+            . ' AND ' . $this->getTableName() . '.uid = ' . (int) $identifier
+        );
+
+        return $record ?: [];
+    }
+
+    public function filterRecordsByRootLineBlacklist(array &$records)
+    {
+        $records = array_filter(
+            $records,
+            function ($record) {
+                return ! $this->isRecordBlacklistedByRootline($record);
+            }
+        );
+    }
+
+    public function prepareRecord(array &$record)
+    {
+        if (isset($record['uid']) && !isset($record['search_identifier'])) {
+            $record['search_identifier'] = $record['uid'];
+        }
+        if (isset($record[$this->tca['ctrl']['label']]) && !isset($record['search_title'])) {
+            $record['search_title'] = $record[$this->tca['ctrl']['label']];
+        }
+    }
+
+    public function getWhereClause() : string
+    {
+        $whereClause = '1=1'
+            . BackendUtility::BEenableFields($this->tableName)
+            . BackendUtility::deleteClause($this->tableName)
+            . ' AND pages.no_search = 0'
+            ;
+
+        if ($this->tableName !== 'pages') {
+            $whereClause .= BackendUtility::BEenableFields('pages')
+                . BackendUtility::deleteClause('pages')
+                ;
+        }
+
+        $userDefinedWhere = $this->configuration->getIfExists(
+            'indexing.' . $this->getTableName() . '.additionalWhereClause'
+        );
+        if (is_string($userDefinedWhere)) {
+            $whereClause .= ' AND ' . $userDefinedWhere;
+        }
+        if ($this->isBlacklistedRootLineConfigured()) {
+            $whereClause .= ' AND pages.uid NOT IN ('
+                . implode(',', $this->getBlacklistedRootLine())
+                . ')'
+                . ' AND pages.pid NOT IN ('
+                . implode(',', $this->getBlacklistedRootLine())
+                . ')';
+        }
+
+        $this->logger->debug('Generated where clause.', [$this->tableName, $whereClause]);
+        return $whereClause;
+    }
+
+    public function getFields() : string
+    {
+        $fields = array_merge(
+            ['uid','pid'],
+            array_filter(
+                array_keys($this->tca['columns']),
+                function ($columnName) {
+                    return !$this->isSystemField($columnName)
+                        && !$this->isUserField($columnName)
+                        && !$this->isPassthroughField($columnName)
+                        ;
+                }
+            )
+        );
+
+        foreach ($fields as $key => $field) {
+            $fields[$key] = $this->tableName . '.' . $field;
+        }
+
+        $this->logger->debug('Generated fields.', [$this->tableName, $fields]);
+        return implode(',', $fields);
+    }
+
+
+    /**
+     * Generate SQL for TYPO3 as a system, to make sure only available records
+     * are fetched.
+     */
+    protected function getSystemWhereClause() : string
+    {
+        $whereClause = '1=1'
+            . BackendUtility::BEenableFields($this->tableName)
+            . BackendUtility::deleteClause($this->tableName)
+            . ' AND pages.no_search = 0'
+            ;
+
+        if ($this->tableName !== 'pages') {
+            $whereClause .= BackendUtility::BEenableFields('pages')
+                . BackendUtility::deleteClause('pages')
+                ;
+        }
+
+        return $whereClause;
+    }
+
+    protected function isSystemField(string $columnName) : bool
+    {
+        $systemFields = [
+            // Versioning fields,
+            // https://docs.typo3.org/typo3cms/TCAReference/Reference/Ctrl/Index.html#versioningws
+            't3ver_oid', 't3ver_id', 't3ver_label', 't3ver_wsid',
+            't3ver_state', 't3ver_stage', 't3ver_count', 't3ver_tstamp',
+            't3ver_move_id', 't3ver_swapmode',
+            $this->tca['ctrl']['transOrigDiffSourceField'],
+            $this->tca['ctrl']['cruser_id'],
+            $this->tca['ctrl']['fe_cruser_id'],
+            $this->tca['ctrl']['fe_crgroup_id'],
+            $this->tca['ctrl']['origUid'],
+        ];
+
+        return in_array($columnName, $systemFields);
+    }
+
+    protected function isUserField(string $columnName) : bool
+    {
+        $config = $this->getColumnConfig($columnName);
+        return isset($config['type']) && $config['type'] === 'user';
+    }
+
+    protected function isPassthroughField(string $columnName) : bool
+    {
+        $config = $this->getColumnConfig($columnName);
+        return isset($config['type']) && $config['type'] === 'passthrough';
+    }
+
+    /**
+     * @throws InvalidArgumentException
+     */
+    public function getColumnConfig(string $columnName) : array
+    {
+        if (!isset($this->tca['columns'][$columnName])) {
+            throw new InvalidArgumentException(
+                'Column does not exist.',
+                InvalidArgumentException::COLUMN_DOES_NOT_EXIST
+            );
+        }
+
+        return $this->tca['columns'][$columnName]['config'];
+    }
+
+    public function getLanguageUidColumn() : string
+    {
+        if (!isset($this->tca['ctrl']['languageField'])) {
+            return '';
+        }
+
+        return $this->tca['ctrl']['languageField'];
+    }
+
+    /**
+     * Checks whether the given record was blacklisted by root line.
+     * This can be configured by typoscript as whole root lines can be black listed.
+     *
+     * Also further TYPO3 mechanics are taken into account. Does a valid root
+     * line exist, is page inside a recycler, is inherited start- endtime
+     * excluded, etc.
+     */
+    protected function isRecordBlacklistedByRootline(array &$record) : bool
+    {
+        $pageUid = $record['pid'];
+        if ($this->tableName === 'pages') {
+            $pageUid = $record['uid'];
+        }
+
+        try {
+            $rootline = $this->objectManager->get(RootlineUtility::class, $pageUid)->get();
+        } catch (\RuntimeException $e) {
+            $this->logger->notice(
+                sprintf('Could not fetch rootline for page %u, because: %s', $pageUid, $e->getMessage()),
+                [$record, $e]
+            );
+            return true;
+        }
+
+        foreach ($rootline as $pageInRootLine) {
+            // Check configured black list if present.
+            if ($this->isBlackListedRootLineConfigured()
+                && in_array($pageInRootLine['uid'], $this->getBlackListedRootLine())
+            ) {
+                $this->logger->info(
+                    sprintf(
+                        'Record %u is black listed due to configured root line configuration of page %u.',
+                        $record['uid'],
+                        $pageInRootLine['uid']
+                    ),
+                    [$record, $pageInRootLine]
+                );
+                return true;
+            }
+
+            if ($pageInRootLine['extendToSubpages'] && (
+                ($pageInRootLine['endtime'] > 0 && $pageInRootLine['endtime'] <= time())
+                || ($pageInRootLine['starttime'] > 0 && $pageInRootLine['starttime'] >= time())
+            )) {
+                $this->logger->info(
+                    sprintf(
+                        'Record %u is black listed due to configured timing of parent page %u.',
+                        $record['uid'],
+                        $pageInRootLine['uid']
+                    ),
+                    [$record, $pageInRootLine]
+                );
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * Checks whether any page uids are black listed.
+     */
+    protected function isBlackListedRootLineConfigured() : bool
+    {
+        return (bool) $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist');
+    }
+
+    /**
+     * Get the list of black listed root line page uids.
+     *
+     * @return array<Int>
+     */
+    protected function getBlackListedRootLine() : array
+    {
+        return GeneralUtility::intExplode(
+            ',',
+            $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist')
+        );
+    }
+
+    protected function getConnection() : \TYPO3\CMS\Core\Database\DatabaseConnection
+    {
+        return $GLOBALS['TYPO3_DB'];
+    }
+}

Classes/Domain/Index/TcaIndexer/TcaTableServiceInterface.php

@@ -0,0 +1,46 @@
+<?php
+namespace Codappix\SearchCore\Domain\Index\TcaIndexer;
+
+/*
+ * Copyright (C) 2016  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+interface TcaTableServiceInterface
+{
+    public function getTableName() : string;
+
+    public function getTableClause() : string;
+
+    /**
+     * Filter the given records by root line blacklist settings.
+     */
+    public function filterRecordsByRootLineBlacklist(array &$records);
+
+    public function prepareRecord(array &$record);
+
+    /**
+     * @throws InvalidArgumentException
+     */
+    public function getColumnConfig(string $columnName) : array;
+
+    public function getRecords(int $offset, int $limit) : array;
+
+    public function getRecord(int $identifier) : array;
+
+    public function getLanguageUidColumn() : string;
+}

Classes/Domain/Model/FacetRequest.php

@@ -30,37 +30,27 @@ class FacetRequest implements FacetRequestInterface
     protected $identifier = '';
 
     /**
-     * @var string
+     * @var array
      */
-    protected $field = '';
+    protected $config = [];
 
     /**
-     * TODO: Add validation / exception?
      * As the facets come from configuration this might be a good idea to help
      * integrators find issues.
-     *
-     * @param string $identifier
-     * @param string $field
      */
-    public function __construct($identifier, $field)
+    public function __construct(string $identifier, array $config)
     {
         $this->identifier = $identifier;
-        $this->field = $field;
+        $this->config = $config;
     }
 
-    /**
-     * @return string
-     */
-    public function getIdentifier()
+    public function getIdentifier() : string
     {
         return $this->identifier;
     }
 
-    /**
-     * @return string
-     */
-    public function getField()
+    public function getConfig() : array
     {
-        return $this->field;
+        return $this->config;
     }
 }

Classes/Domain/Model/QueryResultInterfaceStub.php

@@ -0,0 +1,61 @@
+<?php
+namespace Codappix\SearchCore\Domain\Model;
+
+/*
+ * Copyright (C) 2018  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * As we have to stay compatible with QueryResultInterface
+ * of extbase but can and need not to provide all methods,
+ * this stub will provde the non implemented methods to
+ * keep real implementations clean.
+ */
+trait QueryResultInterfaceStub
+{
+    public function getFirst()
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502195121);
+    }
+
+    public function toArray()
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502195135);
+    }
+
+    public function offsetExists($offset)
+    {
+        // Return false to allow Fluid to use appropriate getter methods.
+        return false;
+    }
+
+    public function offsetGet($offset)
+    {
+        throw new \BadMethodCallException('Use getter to fetch properties.', 1502196933);
+    }
+
+    public function offsetSet($offset, $value)
+    {
+        throw new \BadMethodCallException('You are not allowed to modify the result.', 1502196934);
+    }
+
+    public function offsetUnset($offset)
+    {
+        throw new \BadMethodCallException('You are not allowed to modify the result.', 1502196936);
+    }
+}

Classes/Domain/Model/ResultItem.php

@@ -1,5 +1,5 @@
 <?php
-namespace Codappix\SearchCore\Connection\Elasticsearch;
+namespace Codappix\SearchCore\Domain\Model;
 
 /*
  * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
@@ -29,9 +29,25 @@ class ResultItem implements ResultItemInterface
      */
     protected $data = [];
 
-    public function __construct(\Elastica\Result $result)
+    /**
+     * @var string
+     */
+    protected $type = '';
+
+    public function __construct(array $result, string $type)
     {
-        $this->data = $result->getData();
+        $this->data = $result;
+        $this->type = $type;
+    }
+
+    public function getType() : string
+    {
+        return $this->type;
+    }
+
+    public function getPlainData() : array
+    {
+        return $this->data;
     }
 
     public function offsetExists($offset)

Classes/Domain/Model/SearchRequest.php

@@ -23,6 +23,7 @@ namespace Codappix\SearchCore\Domain\Model;
 use Codappix\SearchCore\Connection\ConnectionInterface;
 use Codappix\SearchCore\Connection\FacetRequestInterface;
 use Codappix\SearchCore\Connection\SearchRequestInterface;
+use Codappix\SearchCore\Domain\Search\SearchService;
 
 /**
  * Represents a search request used to process an actual search.
@@ -64,25 +65,24 @@ class SearchRequest implements SearchRequestInterface
     protected $connection = null;
 
     /**
-     * @param string $query
+     * @var SearchService
      */
-    public function __construct($query = '')
-    {
-        $this->query = (string) $query;
-    }
+    protected $searchService = null;
 
     /**
-     * @return string
+     * @param string $query
      */
-    public function getQuery()
+    public function __construct(string $query = '')
+    {
+        $this->query = $query;
+    }
+
+    public function getQuery() : string
     {
         return $this->query;
     }
 
-    /**
-     * @return string
-     */
-    public function getSearchTerm()
+    public function getSearchTerm() : string
     {
         return $this->query;
     }
@@ -96,26 +96,18 @@ class SearchRequest implements SearchRequestInterface
         $this->filter = \TYPO3\CMS\Extbase\Utility\ArrayUtility::removeEmptyElementsRecursively($filter);
     }
 
-    /**
-     * @return bool
-     */
-    public function hasFilter()
+    public function hasFilter() : bool
     {
         return count($this->filter) > 0;
     }
 
-    /**
-     * @return array
-     */
-    public function getFilter()
+    public function getFilter() : array
     {
         return $this->filter;
     }
 
     /**
      * Add a facet to gather in this search request.
-     *
-     * @param FacetRequestInterface $facet
      */
     public function addFacet(FacetRequestInterface $facet)
     {
@@ -124,10 +116,8 @@ class SearchRequest implements SearchRequestInterface
 
     /**
      * Returns all configured facets to fetch in this search request.
-     *
-     * @return array
      */
-    public function getFacets()
+    public function getFacets() : array
     {
         return $this->facets;
     }
@@ -135,36 +125,49 @@ class SearchRequest implements SearchRequestInterface
     /**
      * Define connection to use for this request.
      * Necessary to allow implementation of execute for interface.
-     *
-     * @param ConnectionInterface $connection
      */
     public function setConnection(ConnectionInterface $connection)
     {
         $this->connection = $connection;
     }
 
+    public function setSearchService(SearchService $searchService)
+    {
+        $this->searchService = $searchService;
+    }
+
     // Extbase QueryInterface
     // Current implementation covers only paginate widget support.
     public function execute($returnRawQueryResult = false)
     {
-        if ($this->connection instanceof ConnectionInterface) {
-            return $this->connection->search($this);
+        if (! ($this->connection instanceof ConnectionInterface)) {
+            throw new \InvalidArgumentException(
+                'Connection was not set before, therefore execute can not work. Use `setConnection` before.',
+                1502197732
+            );
+        }
+        if (! ($this->searchService instanceof SearchService)) {
+            throw new \InvalidArgumentException(
+                'SearchService was not set before, therefore execute can not work. Use `setSearchService` before.',
+                1520325175
+            );
         }
 
-        throw new \InvalidArgumentException(
-            'Connection was not set before, therefore execute can not work. Use `setConnection` before.',
-            1502197732
-        );
+        return $this->searchService->processResult($this->connection->search($this));
     }
 
     public function setLimit($limit)
     {
         $this->limit = (int) $limit;
+
+        return $this;
     }
 
     public function setOffset($offset)
     {
         $this->offset = (int) $offset;
+
+        return $this;
     }
 
     public function getLimit()

Classes/Domain/Model/SearchResult.php

@@ -0,0 +1,129 @@
+<?php
+namespace Codappix\SearchCore\Domain\Model;
+
+/*
+ * Copyright (C) 2016  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\SearchCore\Connection\ResultItemInterface;
+use Codappix\SearchCore\Connection\SearchResultInterface;
+use Codappix\SearchCore\Domain\Model\QueryResultInterfaceStub;
+
+/**
+ * Generic model for mapping a concrete search result from a connection.
+ */
+class SearchResult implements SearchResultInterface
+{
+    use QueryResultInterfaceStub;
+
+    /**
+     * @var SearchResultInterface
+     */
+    protected $originalSearchResult;
+
+    /**
+     * @var array
+     */
+    protected $resultItems = [];
+
+    /**
+     * @var array
+     */
+    protected $results = [];
+
+    /**
+     * For Iterator interface.
+     *
+     * @var int
+     */
+    protected $position = 0;
+
+    public function __construct(SearchResultInterface $originalSearchResult, array $resultItems)
+    {
+        $this->originalSearchResult = $originalSearchResult;
+        $this->resultItems = $resultItems;
+    }
+
+    /**
+     * @return array<ResultItemInterface>
+     */
+    public function getResults() : array
+    {
+        $this->initResults();
+
+        return $this->results;
+    }
+
+    protected function initResults()
+    {
+        if ($this->results !== []) {
+            return;
+        }
+
+        foreach ($this->resultItems as $item) {
+            $this->results[] = new ResultItem($item['data'], $item['type']);
+        }
+    }
+
+    public function getFacets() : array
+    {
+        return $this->originalSearchResult->getFacets();
+    }
+
+    public function getCurrentCount() : int
+    {
+        return $this->originalSearchResult->getCurrentCount();
+    }
+
+    public function count()
+    {
+        return $this->originalSearchResult->count();
+    }
+
+    public function current()
+    {
+        return $this->getResults()[$this->position];
+    }
+
+    public function next()
+    {
+        ++$this->position;
+
+        return $this->current();
+    }
+
+    public function key()
+    {
+        return $this->position;
+    }
+
+    public function valid()
+    {
+        return isset($this->getResults()[$this->position]);
+    }
+
+    public function rewind()
+    {
+        $this->position = 0;
+    }
+
+    public function getQuery()
+    {
+        return $this->originalSearchResult->getQuery();
+    }
+}

Classes/Domain/Search/QueryFactory.php

@@ -23,8 +23,6 @@ namespace Codappix\SearchCore\Domain\Search;
 use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
 use Codappix\SearchCore\Configuration\ConfigurationUtility;
 use Codappix\SearchCore\Configuration\InvalidArgumentException;
-use Codappix\SearchCore\Connection\ConnectionInterface;
-use Codappix\SearchCore\Connection\Elasticsearch\Query;
 use Codappix\SearchCore\Connection\SearchRequestInterface;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 use TYPO3\CMS\Extbase\Utility\ArrayUtility;
@@ -121,6 +119,10 @@ class QueryFactory
             return;
         }
 
+        if (trim($searchRequest->getSearchTerm()) === '') {
+            return;
+        }
+
         $boostQueryParts = [];
 
         foreach ($fields as $fieldName => $boostValue) {
@@ -134,13 +136,15 @@ class QueryFactory
             ];
         }
 
-        $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [
-            'query' => [
-                'bool' => [
-                    'should' => $boostQueryParts,
+        if (!empty($boostQueryParts)) {
+            $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [
+                'query' => [
+                    'bool' => [
+                        'should' => $boostQueryParts,
+                    ],
                 ],
-            ],
-        ]);
+            ]);
+        }
     }
 
     protected function addFactorBoost(array &$query)
@@ -161,7 +165,11 @@ class QueryFactory
     {
         try {
             $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [
-                'stored_fields' => GeneralUtility::trimExplode(',', $this->configuration->get('searching.fields.stored_fields'), true),
+                'stored_fields' => GeneralUtility::trimExplode(
+                    ',',
+                    $this->configuration->get('searching.fields.stored_fields'),
+                    true
+                ),
             ]);
         } catch (InvalidArgumentException $e) {
             // Nothing configured
@@ -169,7 +177,10 @@ class QueryFactory
 
         try {
             $scriptFields = $this->configuration->get('searching.fields.script_fields');
-            $scriptFields = $this->configurationUtility->replaceArrayValuesWithRequestContent($searchRequest, $scriptFields);
+            $scriptFields = $this->configurationUtility->replaceArrayValuesWithRequestContent(
+                $searchRequest,
+                $scriptFields
+            );
             $scriptFields = $this->configurationUtility->filterByCondition($scriptFields);
             if ($scriptFields !== []) {
                 $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['script_fields' => $scriptFields]);
@@ -231,6 +242,18 @@ class QueryFactory
             }
         }
 
+        if (isset($config['raw'])) {
+            $filter = array_merge($config['raw'], $filter);
+        }
+
+        if ($config['type'] === 'range') {
+            return [
+                'range' => [
+                    $config['field'] => $filter,
+                ],
+            ];
+        }
+
         return [$config['field'] => $filter];
     }
 
@@ -239,11 +262,7 @@ class QueryFactory
         foreach ($searchRequest->getFacets() as $facet) {
             $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [
                 'aggs' => [
-                    $facet->getIdentifier() => [
-                        'terms' => [
-                            'field' => $facet->getField(),
-                        ],
-                    ],
+                    $facet->getIdentifier() => $facet->getConfig(),
                 ],
             ]);
         }

Classes/Domain/Search/SearchService.php

@@ -25,7 +25,9 @@ use Codappix\SearchCore\Configuration\InvalidArgumentException;
 use Codappix\SearchCore\Connection\ConnectionInterface;
 use Codappix\SearchCore\Connection\SearchRequestInterface;
 use Codappix\SearchCore\Connection\SearchResultInterface;
+use Codappix\SearchCore\DataProcessing\Service as DataProcessorService;
 use Codappix\SearchCore\Domain\Model\FacetRequest;
+use Codappix\SearchCore\Domain\Model\SearchResult;
 use TYPO3\CMS\Core\Utility\ArrayUtility;
 use TYPO3\CMS\Extbase\Object\ObjectManagerInterface;
 
@@ -49,39 +51,44 @@ class SearchService
      */
     protected $objectManager;
 
+    /**
+     * @var DataProcessorService
+     */
+    protected $dataProcessorService;
+
     /**
      * @param ConnectionInterface $connection
      * @param ConfigurationContainerInterface $configuration
      * @param ObjectManagerInterface $objectManager
+     * @param DataProcessorService $dataProcessorService
      */
     public function __construct(
         ConnectionInterface $connection,
         ConfigurationContainerInterface $configuration,
-        ObjectManagerInterface $objectManager
+        ObjectManagerInterface $objectManager,
+        DataProcessorService $dataProcessorService
     ) {
         $this->connection = $connection;
         $this->configuration = $configuration;
         $this->objectManager = $objectManager;
+        $this->dataProcessorService = $dataProcessorService;
     }
 
-    /**
-     * @param SearchRequestInterface $searchRequest
-     * @return SearchResultInterface
-     */
-    public function search(SearchRequestInterface $searchRequest)
+    public function search(SearchRequestInterface $searchRequest) : SearchResultInterface
     {
-        $searchRequest->setConnection($this->connection);
         $this->addSize($searchRequest);
         $this->addConfiguredFacets($searchRequest);
         $this->addConfiguredFilters($searchRequest);
 
-        return $this->connection->search($searchRequest);
+        // Add connection to request to enable paginate widget support
+        $searchRequest->setConnection($this->connection);
+        $searchRequest->setSearchService($this);
+
+        return $this->processResult($this->connection->search($searchRequest));
     }
 
     /**
      * Add configured size of search result items to request.
-     *
-     * @param SearchRequestInterface $searchRequest
      */
     protected function addSize(SearchRequestInterface $searchRequest)
     {
@@ -92,8 +99,6 @@ class SearchService
 
     /**
      * Add facets from configuration to request.
-     *
-     * @param SearchRequestInterface $searchRequest
      */
     protected function addConfiguredFacets(SearchRequestInterface $searchRequest)
     {
@@ -103,23 +108,16 @@ class SearchService
         }
 
         foreach ($facetsConfig as $identifier => $facetConfig) {
-            if (!isset($facetConfig['field']) || trim($facetConfig['field']) === '') {
-                // TODO: Finish throw
-                throw new \Exception('message', 1499171142);
-            }
-
             $searchRequest->addFacet($this->objectManager->get(
                 FacetRequest::class,
                 $identifier,
-                $facetConfig['field']
+                $facetConfig
             ));
         }
     }
 
     /**
      * Add filters from configuration, e.g. flexform or TypoScript.
-     *
-     * @param SearchRequestInterface $searchRequest
      */
     protected function addConfiguredFilters(SearchRequestInterface $searchRequest)
     {
@@ -138,4 +136,33 @@ class SearchService
             // Nothing todo, no filter configured.
         }
     }
+
+    /**
+     * Processes the result, e.g. applies configured data processing to result.
+     */
+    public function processResult(SearchResultInterface $searchResult) : SearchResultInterface
+    {
+        try {
+            $newSearchResultItems = [];
+            foreach ($this->configuration->get('searching.dataProcessing') as $configuration) {
+                foreach ($searchResult as $resultItem) {
+                    $newSearchResultItems[] = [
+                        'data' => $this->dataProcessorService->executeDataProcessor(
+                            $configuration,
+                            $resultItem->getPlainData()
+                        ),
+                        'type' => $resultItem->getType(),
+                    ];
+                }
+            }
+
+            return $this->objectManager->get(
+                SearchResult::class,
+                $searchResult,
+                $newSearchResultItems
+            );
+        } catch (InvalidArgumentException $e) {
+            return $searchResult;
+        }
+    }
 }

Classes/Domain/Service/DataHandler.php

@@ -22,10 +22,9 @@ namespace Codappix\SearchCore\Domain\Service;
 
 use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
 use Codappix\SearchCore\Domain\Index\IndexerFactory;
+use Codappix\SearchCore\Domain\Index\IndexerInterface;
 use Codappix\SearchCore\Domain\Index\NoMatchingIndexerException;
-use Codappix\SearchCore\Domain\Index\TcaIndexer;
 use TYPO3\CMS\Core\SingletonInterface as Singleton;
-use TYPO3\CMS\Core\Utility\GeneralUtility;
 
 /**
  * Handles all data related things like updates, deletes and inserts.
@@ -83,51 +82,27 @@ class DataHandler implements Singleton
         $this->indexerFactory = $indexerFactory;
     }
 
-    /**
-     * @param string $table
-     * @param array $record
-     */
-    public function add($table, array $record)
-    {
-        $this->logger->debug('Record received for add.', [$table, $record]);
-        $this->getIndexer($table)->indexDocument($record['uid']);
-    }
-
-    /**
-     * @param string $table
-     */
-    public function update($table, array $record)
+    public function update(string $table, array $record)
     {
         $this->logger->debug('Record received for update.', [$table, $record]);
         $this->getIndexer($table)->indexDocument($record['uid']);
     }
 
-    /**
-     * @param string $table
-     * @param int $identifier
-     */
-    public function delete($table, $identifier)
+    public function delete(string $table, string $identifier)
     {
         $this->logger->debug('Record received for delete.', [$table, $identifier]);
         $this->connection->deleteDocument($table, $identifier);
     }
 
     /**
-     * @param string $table
-     * @return IndexerInterface
-     *
      * @throws NoMatchingIndexerException
      */
-    protected function getIndexer($table)
+    protected function getIndexer(string $table) : IndexerInterface
     {
         return $this->indexerFactory->getIndexer($table);
     }
 
-    /**
-     * @param string $table
-     * @return bool
-     */
-    public function canHandle($table)
+    public function supportsTable(string $table) : bool
     {
         try {
             $this->getIndexer($table);
@@ -135,7 +110,5 @@ class DataHandler implements Singleton
         } catch (NoMatchingIndexerException $e) {
             return false;
         }
-
-        return false;
     }
 }

Classes/Hook/DataHandler.php

@@ -21,7 +21,6 @@ namespace Codappix\SearchCore\Hook;
  */
 
 use Codappix\SearchCore\Configuration\NoConfigurationException;
-use Codappix\SearchCore\Domain\Index\NoMatchingIndexerException;
 use Codappix\SearchCore\Domain\Service\DataHandler as OwnDataHandler;
 use TYPO3\CMS\Backend\Utility\BackendUtility;
 use TYPO3\CMS\Core\DataHandling\DataHandler as CoreDataHandler;
@@ -49,99 +48,103 @@ class DataHandler implements Singleton
     /**
      * Dependency injection as TYPO3 doesn't provide it on it's own.
      * Still you can submit your own dataHandler.
-     *
-     * @param OwnDataHandler $dataHandler
-     * @param Logger $logger
      */
     public function __construct(OwnDataHandler $dataHandler = null, Logger $logger = null)
     {
-        $this->dataHandler = $dataHandler;
-        if ($this->dataHandler === null) {
+        if ($dataHandler === null) {
             try {
-                $this->dataHandler = GeneralUtility::makeInstance(ObjectManager::class)
+                $dataHandler = GeneralUtility::makeInstance(ObjectManager::class)
                     ->get(OwnDataHandler::class);
             } catch (NoConfigurationException $e) {
                 // We have no configuration. That's fine, hooks will not be
                 // executed due to check for existing DataHandler.
             }
         }
+        $this->dataHandler = $dataHandler;
 
-        $this->logger = $logger;
-        if ($this->logger === null) {
-            $this->logger = GeneralUtility::makeInstance(LogManager::class)
+        if ($logger === null) {
+            $logger = GeneralUtility::makeInstance(LogManager::class)
                 ->getLogger(__CLASS__);
         }
+        $this->logger = $logger;
     }
 
     /**
      * Called by CoreDataHandler on deletion of records.
-     *
-     * @param string $table
-     * @param int $uid
-     *
-     * @return bool False if hook was not processed.
      */
-    public function processCmdmap_deleteAction($table, $uid)
+    public function processCmdmap_deleteAction(string $table, string $uid) : bool
     {
         if (! $this->shouldProcessHookForTable($table)) {
             $this->logger->debug('Delete not processed.', [$table, $uid]);
             return false;
         }
 
-        $this->dataHandler->delete($table, $uid);
+        $this->dataHandler->delete($table, (string) $uid);
         return true;
     }
 
-    /**
-     * Called by CoreDataHandler on database operations, e.g. if new records were created or records were updated.
-     *
-     * @param string $status
-     * @param string $table
-     * @param string|int $uid
-     * @param array $fieldArray
-     * @param CoreDataHandler $dataHandler
-     *
-     * @return bool False if hook was not processed.
-     */
-    public function processDatamap_afterDatabaseOperations($status, $table, $uid, array $fieldArray, CoreDataHandler $dataHandler)
+    public function processDatamap_afterAllOperations(CoreDataHandler $dataHandler)
+    {
+        foreach ($dataHandler->datamap as $table => $record) {
+            $uid = key($record);
+            $fieldData = current($record);
+
+            if (isset($fieldData['uid'])) {
+                $uid = $fieldData['uid'];
+            } elseif (isset($dataHandler->substNEWwithIDs[$uid])) {
+                $uid = $dataHandler->substNEWwithIDs[$uid];
+            }
+
+            if (!is_numeric($uid) || $uid <= 0) {
+                continue;
+            }
+
+            $this->processRecord($table, $uid);
+        }
+    }
+
+    public function clearCachePostProc(array $parameters, CoreDataHandler $dataHandler)
+    {
+        $pageUid = 0;
+
+        // If editor uses "small page blizzard"
+        if (isset($parameters['cacheCmd']) && is_numeric($parameters['cacheCmd'])) {
+            $pageUid = $parameters['cacheCmd'];
+        }
+        // If records were changed
+        if (isset($parameters['uid_page']) && is_numeric($parameters['uid_page'])) {
+            $pageUid = $parameters['uid_page'];
+        }
+
+        if ($pageUid > 0) {
+            $this->processRecord('pages', (int) $pageUid);
+        }
+    }
+
+    protected function processRecord(string $table, int $uid) : bool
     {
         if (! $this->shouldProcessHookForTable($table)) {
-            $this->logger->debug('Database update not processed.', [$table, $uid]);
+            $this->logger->debug('Indexing of record not processed.', [$table, $uid]);
             return false;
         }
 
-        if ($status === 'new') {
-            $fieldArray['uid'] = $dataHandler->substNEWwithIDs[$uid];
-            $this->dataHandler->add($table, $fieldArray);
+        $record = $this->getRecord($table, $uid);
+        if ($record !== null) {
+            $this->dataHandler->update($table, $record);
             return true;
         }
 
-        if ($status === 'update') {
-            $record = $this->getRecord($table, $uid);
-            if ($record !== null) {
-                $this->dataHandler->update($table, $record);
-            }
-            return true;
-        }
-
-        $this->logger->debug(
-            'Database update not processed, cause status is unhandled.',
-            [$status, $table, $uid, $fieldArray]
-        );
+        $this->logger->debug('Indexing of record not processed, as he was not found in Database.', [$table, $uid]);
         return false;
     }
 
-    /**
-     * @param string $table
-     * @return bool
-     */
-    protected function shouldProcessHookForTable($table)
+    protected function shouldProcessHookForTable(string $table) : bool
     {
         if ($this->dataHandler === null) {
             $this->logger->debug('Datahandler could not be setup.');
             return false;
         }
-        if (! $this->dataHandler->canHandle($table)) {
+        if (! $this->dataHandler->supportsTable($table)) {
             $this->logger->debug('Table is not allowed.', [$table]);
             return false;
         }
@@ -152,11 +155,9 @@ class DataHandler implements Singleton
     /**
      * Wrapper to allow unit testing.
      *
-     * @param string $table
-     * @param int $uid
-     * @return null|array<String>
+     * @return array|null
      */
-    protected function getRecord($table, $uid)
+    protected function getRecord(string $table, int $uid)
     {
         return BackendUtility::getRecord($table, $uid);
     }

Classes/Integration/Form/Finisher/DataHandlerFinisher.php

@@ -58,13 +58,11 @@ class DataHandlerFinisher extends AbstractFinisher
 
         switch ($action) {
             case 'update':
+            case 'add':
                 $this->dataHandler->update($tableName, $record);
                 break;
-            case 'add':
-                $this->dataHandler->add($tableName, $record);
-                break;
             case 'delete':
-                $this->dataHandler->delete($tableName, $record['uid']);
+                $this->dataHandler->delete($tableName, (string) $record['uid']);
                 break;
         }
     }

Classes/Utility/FrontendUtility.php

@@ -29,10 +29,7 @@ use TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController;
  */
 class FrontendUtility extends BackendUtility
 {
-    /**
-     * @return TypoScriptFrontendController
-     */
-    protected static function getLanguageService()
+    protected static function getLanguageService() : TypoScriptFrontendController
     {
         return $GLOBALS['TSFE'];
     }

Configuration/TypoScript/constants.txt

@@ -11,12 +11,13 @@ plugin {
 
             indexing {
                 tt_content {
-                    additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu', 'shortcut', 'search', 'login') AND tt_content.bodytext != ''
+                    additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu', 'shortcut', 'search', 'login') AND (tt_content.bodytext != '' OR tt_content.header != '')
                 }
 
                 pages {
                     additionalWhereClause = pages.doktype NOT IN (3, 199, 6, 254, 255)
                     abstractFields = abstract, description, bodytext
+                    contentFields = header, bodytext
                 }
             }
         }

Configuration/TypoScript/setup.txt

@@ -20,6 +20,7 @@ plugin {
                     indexer = Codappix\SearchCore\Domain\Index\TcaIndexer\PagesIndexer
                     additionalWhereClause = {$plugin.tx_searchcore.settings.indexing.pages.additionalWhereClause}
                     abstractFields = {$plugin.tx_searchcore.settings.indexing.pages.abstractFields}
+                    contentFields = {$plugin.tx_searchcore.settings.indexing.pages.contentFields}
                 }
             }
 

Documentation/source/changelog.rst

@@ -0,0 +1,14 @@
+Changelog
+=========
+
+.. toctree::
+   :maxdepth: 1
+   :glob:
+
+   changelog/20180424-149-extract-relation-resolver-to-data-processing
+   changelog/20180410-148-keep-sys_language_uid
+   changelog/20180315-134-make-conent-fields-configurable
+   changelog/20180309-25-provide-sys-language-uid
+   changelog/20180308-131-respect-page-cache-clear
+   changelog/20180308-introduce-php70-type-hints
+   changelog/20180306-120-facet-configuration

Documentation/source/changelog/20180306-120-facet-configuration.rst

@@ -0,0 +1,40 @@
+Breaking Change 120 "Pass facets configuration to Elasticsearch"
+================================================================
+
+In order to allow arbitrary facet configuration, we do not process the facet configuration anymore.
+Instead integrators are able to configure facets for search service "as is". We just pipe the
+configuration through.
+
+Therefore the following, which worked before, does not work anymore:
+
+.. code-block:: typoscript
+   :linenos:
+   :emphasize-lines: 4
+
+    plugin.tx_searchcore.settings.search {
+        facets {
+            category {
+                field = categories
+            }
+        }
+    }
+
+Instead you have to provide the full configuration yourself:
+
+.. code-block:: typoscript
+   :linenos:
+   :emphasize-lines: 4,6
+
+    plugin.tx_searchcore.settings.search {
+        facets {
+            category {
+                terms {
+                    field = categories
+                }
+            }
+        }
+    }
+
+You need to add line 4 and 6, the additional level ``terms`` for Elasticsearch.
+
+See :issue:`120`.

Documentation/source/changelog/20180308-131-respect-page-cache-clear.rst

@@ -0,0 +1,14 @@
+Feature 131 "Pages do not get indexed if content has changed"
+=============================================================
+
+Previously we only used DataHandler hooks triggered when processing records. This way we did not
+index a page when content has changed.
+
+We now also use cache clear hooks of DataHandler to index pages whenever their cache get cleared.
+This way we also index a page if an integrator configured to clear further pages if content was
+changed.
+
+Still there are limitations. We do not get informed for pages which got cleared due to attached
+caches via TypoScript.
+
+See :issue:`131`.

Documentation/source/changelog/20180308-introduce-php70-type-hints.rst

@@ -0,0 +1,12 @@
+Breaking Change "Introduce PHP 7.0 TypeHints"
+=============================================
+
+As PHP evolved, we now migrate the whole code base to use PHP 7.0 type hints.
+We do not use PHP 7.1 Type Hints, as some customers still need PHP 7.0 support.
+
+Also we added missing methods to interfaces, that were already used in code.
+
+As this leads to changed method signatures, most custom implementations of interfaces, or overwrites
+of existing methods are broken.
+
+To fix, just update the signatures as pointed out by PHP while running the code.

Documentation/source/changelog/20180309-25-provide-sys-language-uid.rst

@@ -0,0 +1,16 @@
+Feature 25 "Respect multiple languages" - Provide sys_language_uid
+==================================================================
+
+Previously we did not fetch ``sys_language_uid`` field from database. This prevented everyone from
+working with multiple languages.
+By not removing the field it gets indexed and provides a very basic way of implementing multiple
+languages.
+At least it's now possible to filter search results by current language for now. Still the records
+are not "valid" as we do not add overlays for now.
+
+This is a first step into full multi language support.
+
+Martin Hummer already has a basic proof of concept, based on :ref:`concepts_dataprocessing` working,
+depending on ``sys_language_uid``.
+
+See :issue:`25`.

Documentation/source/changelog/20180315-134-make-conent-fields-configurable.rst

@@ -0,0 +1,13 @@
+Feature 134 "Enable indexing of tt_content records of CType Header"
+===================================================================
+
+Before, only ``bodytext`` was used to generate content while indexing pages.
+
+As there are content elements like ``header`` where this field is empty, but content is still
+available, it's now possible to configure the fields.
+This makes it also possible to configure further custom content elements with new columns.
+
+A new TypoScript option is now available, and ``header`` is added by default, see
+:ref:`contentFields`.
+
+See :issue:`134`.

Documentation/source/changelog/20180410-148-keep-sys_language_uid.rst

@@ -0,0 +1,10 @@
+Feature 148 "Cast sys_language_uid to int"
+==========================================
+
+While resolving relations the configured language uid field, fetched from TCA, will
+be casted to integer and returned immediately.
+
+This change prevents the bug mentioned in :issue:`148`, where `0` is casted to an
+empty string, which makes filtering hard.
+
+See :issue:`148`.

Documentation/source/changelog/20180424-149-extract-relation-resolver-to-data-processing.rst

@@ -0,0 +1,24 @@
+Breaking Change 149 "Extract RelationResolver to a new DataProcessor"
+=====================================================================
+
+The resolving of relation, based on TCA, is no longer done by the indexer. Instead we
+now provide a DataProcessor to solve this job.
+
+As this makes it necessary to configure the DataProcessor, this is a breaking change.
+Before the resolving was done out of the box.
+
+So why did we change that? The resolving of relations was already implemented before
+we added the data processors. As the concept of data processors is far more flexible,
+we try to migrate hard coupled components step by step. The benefit of this change is
+basically that you can now configure the resolving of relations and far more
+important, the order of execution.
+
+Now it's possible to first copy some fields, e.g. ``starttime`` and ``endtime`` to
+further fields and to resolve relations afterwards. As the copied fields are not
+configured in TCA, they will be skipped. This way an integrator can keep certain
+information as they are.
+
+Also the processor is now configured as all other processors. You can now optionally
+configure fields to not process.
+
+See :issue:`149` and :issue:`147`.

Documentation/source/concepts.rst

@@ -3,38 +3,47 @@
 Concepts
 ========
 
-The extension is built with the following concepts in mind.
+The main concept is to provide a foundation where other developers can profit from, to provide
+integrations into search services like Elasticsearch, Algolia, ... .
+
+Our code contains the following concepts which should be understand:
 
 .. _concepts_connections:
 
 Connections
 -----------
 
-It should be possible to use different search services like elasticsearch and solr out of the box.
-If a service is not contained, it should be possible to implement the necessary part by implementing
-the necessary interfaces and configuring the extension to use the new connection.
+Different search services can provide integrations. ``search_core`` only provides abstractions and
+interfaces. The main purpose is to provide a stable API between TYPO3 and concrete connection.
 
-Also it should be possible to use multiple connections at once. This way multiple search services
-can be used in the same installation.
-
-Currently only :ref:`Elasticsearch` is provided.
+For information about implementing a new connection, take a look at :ref:`development_connection`.
 
 .. _concepts_indexing:
 
 Indexing
 --------
 
-The indexing is done by one of the available indexer. For each identifier it's possible to define
-the indexer to use. Also it's possible to write custom indexer to use.
+Indexing is the process of collecting and preparing data, before sending it to a Connection.
+The indexing is done by one of the available indexer. Indexer are identified by a key, as configured
+in TypoScript.
 
-Currently only the :ref:`TcaIndexer` is provided.
+Currently :ref:`TcaIndexer` and :ref:`PagesIndexer` are provided.
 
-.. _concepts_indexing_dataprocessing:
+For information about implementing a new indexer, take a look at :ref:`development_indexer`.
+
+.. _concepts_dataprocessing:
 
 DataProcessing
 ^^^^^^^^^^^^^^
 
 Before data is transfered to search service, it can be processed by "DataProcessors" like already
 known by :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing` of :ref:`t3tsref:cobj-fluidtemplate`.
+The same is true for retrieved search results. They can be processed again by "DataProcessors" to
+prepare data for display in Templates or further usage.
 
-Configuration is done through TypoScript, see :ref:`dataProcessing`.
+This should keep indexers simple and move logic to DataProcessors. This makes most parts highly
+flexible as integrators are able to configure DataProcessors and change their order.
+
+Configuration is done through TypoScript, see :ref:`dataprocessors`.
+
+For information about implementing a new DataProcessor, take a look at :ref:`development_dataprocessor`.

Documentation/source/conf.py

@@ -51,7 +51,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'TYPO3 Extension search_core'
-copyright = u'2016, Daniel Siepmann'
+copyright = u'2016 - 2018, Daniel Siepmann'
 author = u'Daniel Siepmann'
 
 # The version info for the project you're documenting, acts as replacement for
@@ -59,10 +59,9 @@ author = u'Daniel Siepmann'
 # built documents.
 #
 # The short X.Y version.
-version = u'1.0.0'
+version = u'0.0.4'
 # The full version, including alpha/beta/rc tags.
-release = u'1.0.0'
-
+release = u'0.0.4'
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
 #
@@ -133,7 +132,7 @@ html_theme_options = {
 
 # The name for this set of Sphinx documents.
 # "<project> v<release> documentation" by default.
-#html_title = u'TYPO3 Extension search_core v1.0.0'
+#html_title = u'TYPO3 Extension search_core v0.0.4'
 
 # A shorter title for the navigation bar.  Default is the same as html_title.
 #html_short_title = None
@@ -150,7 +149,7 @@ html_theme_options = {
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+# html_static_path = ['_static']
 
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied

Documentation/source/configuration.rst

@@ -5,9 +5,12 @@
 Configuration
 =============
 
-The extension offers the following configuration options through TypoScript. If you overwrite them
-through `setup` make sure to keep them in the `module` area as they will be accessed from backend
-mode of TYPO3. Do so by placing the following line at the end::
+Installation wide configuration is handled inside of the extension manager. Just check out the
+options there, they all have labels.
+
+Everything else is configured through TypoScript. If you overwrite them through `setup` make sure to
+keep them in the `module` area as they will be accessed from backend mode of TYPO3 for indexing. Do
+so by placing the following line at the end::
 
     module.tx_searchcore < plugin.tx_searchcore
 
@@ -23,12 +26,10 @@ Here is the example default configuration that's provided through static include
 
 .. literalinclude:: ../../Configuration/TypoScript/constants.txt
    :language: typoscript
-   :linenos:
    :caption: Static TypoScript Constants
 
 .. literalinclude:: ../../Configuration/TypoScript/setup.txt
    :language: typoscript
-   :linenos:
    :caption: Static TypoScript Setup
 
 .. _configuration_options:

Documentation/source/configuration/connections.rst

@@ -4,7 +4,7 @@ Connections
 ===========
 
 Holds settings regarding the different possible connections for search services like Elasticsearch
-or Solr.
+or Algolia.
 
 Configured as::
 
@@ -22,15 +22,13 @@ Configured as::
 
 Where ``connectionName`` is one of the available :ref:`connections`.
 
-The following settings are available. For each setting its documented which connection consumes it.
+The following settings are available.
 
 .. _host:
 
 ``host``
 --------
 
-Used by: :ref:`Elasticsearch`.
-
 The host, e.g. ``localhost`` or an IP where the search service is reachable from TYPO3
 installation.
 
@@ -43,8 +41,6 @@ Example::
 ``port``
 --------
 
-Used by: :ref:`Elasticsearch`.
-
 The port where search service is reachable. E.g. default ``9200`` for Elasticsearch.
 
 Example::
@@ -63,6 +59,3 @@ The index where the documents are being indexed to. E.g. default ``typo3content`
 Example::
 
     plugin.tx_searchcore.settings.connections.elasticsearch.index = typo3content
-
-
-

Documentation/source/configuration/dataProcessing/ContentObjectDataProcessorAdapterProcessor.rst

@@ -0,0 +1,32 @@
+``Codappix\SearchCore\DataProcessing\ContentObjectDataProcessorAdapterProcessor``
+=================================================================================
+
+Will execute an existing TYPO3 data processor.
+
+Possible Options:
+
+``_dataProcessor``
+    Necessary, defined which data processor to apply. Provide the same as you would to call the
+    processor.
+``_table``
+    Defines the "current" table as used by some processors, e.g.
+    ``TYPO3\CMS\Frontend\DataProcessing\FilesProcessor``.
+
+All further options are passed to the configured data processor. Therefore they are documented at
+each data processor.
+
+Example::
+
+    plugin.tx_searchcore.settings.searching.dataProcessing {
+        1 = Codappix\SearchCore\DataProcessing\ContentObjectDataProcessorAdapterProcessor
+        1 {
+            _table = pages
+            _dataProcessor = TYPO3\CMS\Frontend\DataProcessing\FilesProcessor
+
+            references.fieldName = media
+            as = images
+        }
+    }
+
+The above example will create a new field ``images`` with resolved FAL relations from ``media``
+field.

Documentation/source/configuration/dataProcessing/CopyToProcessor.rst

@@ -8,6 +8,10 @@ Possible Options:
 ``to``
     Defines the field to copy the values into. All values not false will be copied at the moment.
 
+``from``
+    Optional, defines the field to copy, can only be one field.
+    If empty, all existing fields will be copied.
+
 Example::
 
     plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing {
@@ -17,7 +21,8 @@ Example::
         }
         2 = Codappix\SearchCore\DataProcessing\CopyToProcessor
         2 {
-            to = spellcheck
+            from = uid
+            to = backup_uid
         }
     }
 

Documentation/source/configuration/dataProcessing/RemoveProcessor.rst

@@ -1,7 +1,7 @@
 ``Codappix\SearchCore\DataProcessing\RemoveProcessor``
 ======================================================
 
-Will remove fields from record, e.g. if you do not want to sent them to elasticsearch at all.
+Will remove fields from record.
 
 Possible Options:
 

Documentation/source/configuration/dataProcessing/TcaRelationResolvingProcessor.rst

@@ -0,0 +1,34 @@
+``Codappix\SearchCore\DataProcessing\TcaRelationResolvingProcessor``
+====================================================================
+
+Will resolve relations through TCA for record.
+The result will be the same as in list view of TYPO3 Backend. E.g. Check boxes will be
+resolved to their label, dates will be resolved to human readable representation and
+relations will be resolved to their configured labels.
+
+Combine with CopyToProcessor or exclude certain fields to keep original value for
+further processing.
+
+Mandatory Options:
+
+``_table``
+    The TCA table as found on top level of ``$GLOBALS['TCA']``.
+
+    This will auto filled for indexing through the provided indexers. Still you can
+    apply processors on results, where no information about the table exists anymore.
+
+Possible Options:
+
+``excludeFields``
+    Comma separated list of fields to not resolve relations for.
+
+Example::
+
+    plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing {
+        1 = Codappix\SearchCore\DataProcessing\TcaRelationResolvingProcessor
+        1 {
+            _table = tt_content
+            excludeFields = starttime, endtime
+        }
+    }
+

Documentation/source/configuration/indexing.rst

@@ -40,11 +40,7 @@ Contains a comma separated list of page uids. Spaces are trimmed.
 
 Example::
 
-    plugin.tx_searchcore.settings.indexing.<identifier>.rootLineBlacklist = 3, 10, 100
-
-Also it's possible to define some behaviour for the different document types. In context of TYPO3
-tables are used as document types 1:1. It's possible to configure different tables. The following
-options are available:
+    plugin.tx_searchcore.settings.indexing.pages.rootLineBlacklist = 3, 10, 100
 
 .. _additionalWhereClause:
 
@@ -55,16 +51,16 @@ Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`.
 
 Add additional SQL to where clauses to determine indexable records from the table. This way you
 can exclude specific records like ``tt_content`` records with specific ``CType`` values or
-something else. E.g. you can add a new field to the table to exclude records from indexing.
+something else.
 
 Example::
 
-    plugin.tx_searchcore.settings.indexing.<identifier>.additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu')
+    plugin.tx_searchcore.settings.indexing.tt_content.additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu')
 
 .. attention::
 
     Make sure to prefix all fields with the corresponding table name. The selection from
-    database will contain joins and can lead to SQL errors if a field exists in multiple tables.
+    database might contain joins and can lead to SQL errors if a field exists in multiple tables.
 
 .. _abstractFields:
 
@@ -73,6 +69,10 @@ abstractFields
 
 Used by: :ref:`PagesIndexer`.
 
+.. note::
+
+   Will be migrated to :ref:`dataprocessors` in the future.
+
 Define which field should be used to provide the auto generated field "search_abstract".
 The fields have to exist in the record to be indexed. Therefore fields like ``content`` are also
 possible.
@@ -80,21 +80,38 @@ possible.
 Example::
 
     # As last fallback we use the content of the page
-    plugin.tx_searchcore.settings.indexing.<identifier>.abstractFields := addToList(content)
+    plugin.tx_searchcore.settings.indexing.pages.abstractFields := addToList(content)
 
 Default::
 
     abstract, description, bodytext
 
+.. _contentFields:
+
+contentFields
+-------------
+
+Used by: :ref:`PagesIndexer`.
+
+Define which fields should be used to provide the auto generated field "content".
+
+Example::
+
+    plugin.tx_searchcore.settings.indexing.pages.contentFields := addToList(table_caption)
+
+Default::
+
+    header, bodytext
+
 .. _mapping:
 
 mapping
 -------
 
-Used by: Elasticsearch connection while indexing.
+Used by: :ref:`connection_elasticsearch` connection while indexing.
 
 Define mapping for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/mapping.html
-You are able to define the mapping for each property / columns.
+You are able to define the mapping for each property / column.
 
 Example::
 
@@ -112,7 +129,7 @@ makes building a facet possible.
 index
 -----
 
-Used by: Elasticsearch connection while indexing.
+Used by: :ref:`connection_elasticsearch` connection while indexing.
 
 Define index for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/indices-create-index.html
 
@@ -141,67 +158,12 @@ Example::
 
 ``char_filter`` and ``filter`` are a comma separated list of options.
 
-.. _dataProcessing:
+.. _indexing_dataProcessing:
 
 dataProcessing
 --------------
 
-Used by: All connections while indexing.
+Used by: All connections while indexing, due to implementation inside ``AbstractIndexer``.
 
-Configure modifications on each document before sending it to the configured connection. Same as
-provided by TYPO3 for :ref:`t3tsref:cobj-fluidtemplate` through
-:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`.
-
-All processors are applied in configured order. Allowing to work with already processed data.
-
-Example::
-
-    plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing {
-        1 = Codappix\SearchCore\DataProcessing\CopyToProcessor
-        1 {
-            to = search_spellcheck
-        }
-
-        2 = Codappix\SearchCore\DataProcessing\CopyToProcessor
-        2 {
-            to = search_all
-        }
-    }
-
-The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards
-all fields, including ``search_spellcheck`` will be copied to ``search_all``.
-E.g. used to index all information into a field for :ref:`spellchecking` or searching with
-different :ref:`mapping`.
-
-The following Processor are available:
-
-.. toctree::
-    :maxdepth: 1
-    :glob:
-
-    dataProcessing/CopyToProcessor
-    dataProcessing/RemoveProcessor
-    dataProcessing/GeoPointProcessor
-
-The following Processor are planned:
-
-    ``Codappix\SearchCore\DataProcessing\ReplaceProcessor``
-        Will execute a search and replace on configured fields.
-
-    ``Codappix\SearchCore\DataProcessing\RootLevelProcessor``
-        Will attach the root level to the record.
-
-    ``Codappix\SearchCore\DataProcessing\ChannelProcessor``
-        Will add a configurable channel to the record, e.g. if you have different areas in your
-        website like "products" and "infos".
-
-    ``Codappix\SearchCore\DataProcessing\RelationResolverProcessor``
-        Resolves all relations using the TCA.
-
-Of course you are able to provide further processors. Just implement
-``Codappix\SearchCore\DataProcessing\ProcessorInterface`` and use the FQCN (=Fully qualified
-class name) as done in the examples above.
-
-By implementing also the same interface as necessary for TYPO3
-:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`, you are able to reuse the same code
-also for Fluid to prepare the same record fetched from DB for your fluid.
+Configure modifications on each document before sending it to the configured connection.
+For full documentation check out :ref:`dataprocessors`.

Documentation/source/configuration/searching.rst

@@ -8,45 +8,50 @@ Searching
 size
 ----
 
-Used by: Elasticsearch connection while building search query.
-
 Defined how many search results should be fetched to be available in search result.
 
 Example::
 
     plugin.tx_searchcore.settings.searching.size = 50
 
-Default if not configured is 10.
+Default is ``10``.
 
 .. _facets:
 
 facets
 ------
 
-Used by: Elasticsearch connection while building search query.
-
 Define aggregations for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-aggregations-bucket-terms-aggregation.html
-Currently only the term facet is provided.
 
 Example::
 
-    plugin.tx_searchcore.settings.searching.facets {
-        contentTypes {
-            field = CType
+    category {
+        terms {
+            field = categories
         }
     }
 
-The above example will provide a facet with options for all found ``CType`` results together
-with a count.
+    month {
+        date_histogram {
+            field = released
+            interval = month
+            format = Y-MM-01
+            order {
+                _time = desc
+            }
+        }
+    }
+
+
+The above example will provide a facet with options for all found ``categories`` results together
+with a count. Also a facet for ``released`` will be provided.
 
 .. _filter:
 
 filter
 ------
 
-Used by: While building search request.
-
-Define filter that should be set for all requests.
+Define filter that should be set for all search requests.
 
 Example::
 
@@ -54,6 +59,9 @@ Example::
         property = value
     }
 
+Also see :ref:`mapping.filter` to map incoming request information, e.g. from a ``select``, to build
+more complex filters.
+
 For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``keyword``.
 
 .. _minimumShouldMatch:
@@ -61,9 +69,8 @@ For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``key
 minimumShouldMatch
 ------------------
 
-Used by: Elasticsearch connection while building search query.
-
-Define the minimum match for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-minimum-should-match.html
+Define the minimum match for Elasticsearch, have a look at the official docs:
+https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-minimum-should-match.html
 
 Example::
 
@@ -74,8 +81,6 @@ Example::
 boost
 -----
 
-Used by: Elasticsearch connection while building search query.
-
 Define fields that should boost the score for results.
 
 Example::
@@ -93,10 +98,9 @@ https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_boosting_query_clauses.
 fieldValueFactor
 ----------------
 
-Used by: Elasticsearch connection while building search query.
-
-Define a field to use as a factor for scoring. The configuration is passed through to elastic
-search ``field_value_factor``, see: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor
+Define a field to use as a factor for scoring. The configuration is passed through to Elasticsearch
+``field_value_factor``, see:
+https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor
 
 Example::
 
@@ -119,55 +123,84 @@ E.g. you submit a filter in form of:
 
 .. code-block:: html
 
-    <f:form.textfield property="filter.distance.location.lat" value="51.168098" />
-    <f:form.textfield property="filter.distance.location.lon" value="6.381384" />
-    <f:form.textfield property="filter.distance.distance" value="100km" />
+   <f:comment>
+       Due to TYPO3 7.x fluid limitations, we build this input ourself.
+       No longer necessary in 8 and above
+   </f:comment>
+   <select name="tx_searchcore_search[searchRequest][filter][month][from]" class="_control" >
+       <option value="">Month</option>
+       <f:for each="{searchResult.facets.month.options}" as="month">
+           <f:if condition="{month.count}">
+               <option
+                   value="{month.displayName -> f:format.date(format: 'Y-m')}"
+                   {f:if(condition: '{searchRequest.filter.month.from} == {month.displayName -> f:format.date(format: \'Y-m\')}', then: 'selected="true"')}
+               >{month.displayName -> f:format.date(format: '%B %Y')}</option>
+           </f:if>
+       </f:for>
+   </select>
+   <select name="tx_searchcore_search[searchRequest][filter][month][to]" class="_control" >
+       <option value="">Month</option>
+       <f:for each="{searchResult.facets.month.options}" as="month">
+           <f:if condition="{month.count}">
+               <option
+                   value="{month.displayName -> f:format.date(format: 'Y-m')}"
+                   {f:if(condition: '{searchRequest.filter.month.from} == {month.displayName -> f:format.date(format: \'Y-m\')}', then: 'selected="true"')}
+               >{month.displayName -> f:format.date(format: '%B %Y')}</option>
+           </f:if>
+       </f:for>
+   </select>
 
-This will create a ``distance`` filter with subproperties. To make this filter actually work, you
+This will create a ``month`` filter with sub properties. To make this filter actually work, you
 can add the following TypoScript, which will be added to the filter::
 
     mapping {
         filter {
-            distance {
-                field = geo_distance
+            month {
+                type = range
+                field = released
+                raw {
+                    format = yyyy-MM
+                }
+
                 fields {
-                    distance = distance
-                    location = location
+                    gte = from
+                    lte = to
                 }
             }
         }
     }
 
 ``fields`` has a special meaning here. This will actually map the properties of the filter to fields
-in elasticsearch. In above example they do match, but you can also use different names in your form.
-On the left hand side is the elasticsearch field name, on the right side the one submitted as a
-filter.
+in Elasticsearch. On the left hand side is the Elasticsearch field name, on the right side the one
+submitted as a filter.
 
-The ``field``, in above example ``geo_distance``, will be used as the elasticsearch field for
-filtering. This way you can use arbitrary filter names and map them to existing elasticsearch fields.
+The ``field``, in above example ``released``, will be used as the Elasticsearch field for
+filtering. This way you can use arbitrary filter names and map them to existing Elasticsearch fields.
+
+Everything that is configured inside ``raw`` is passed, as is, to search service, e.g.
+Elasticsearch.
 
 .. _fields:
 
 fields
 ------
 
-Defines the fields to fetch and search from elasticsearch. With the following sub keys:
+Defines the fields to fetch and search from Elasticsearch. With the following sub keys:
 
-``query`` defines the fields to search in. Default is ``_all`` from 5.x times of elasticsearch.
-Configure a comma separated list of fields to search in. This is necessary if you have configured
-special mapping for some fields, or just want to search some fields.
-The most hits get ranked highest. The following is an example configuration::
+``query`` defines the fields to search in. Configure a comma separated list of fields to search in.
+This is necessary if you have configured special mapping for some fields, or just want to search
+some fields. The following is an example configuration::
 
     fields {
         query = _all, city
     }
 
-The following sub properties configure the fields to fetch from elasticsearch:
+The following sub properties configure the fields to fetch from Elasticsearch:
 
 First ``stored_fields`` which is a list of comma separated fields which actually exist and will be
 added. Typically you will use ``_source`` to fetch the whole indexed fields.
 
-Second is ``script_fields``, which allow you to configure scripted fields for elasticsearch.
+Second is ``script_fields``, which allow you to configure scripted fields for Elasticsearch.
 An example might look like the following::
 
     fields {
@@ -190,7 +223,7 @@ In above example we add a single ``script_field`` called ``distance``. We add a
 field should be added. The condition will be parsed as Fluidtemplate and is casted to bool via PHP.
 If the condition is true, or no ``condition`` exists, the ``script_field`` will be added to the
 query. The ``condition`` will be removed and everything else is submitted one to one to
-elasticsearch, except each property is run through Fluidtemplate, to allow you to use information
+Elasticsearch, except each property is run through Fluidtemplate, to allow you to use information
 from search request, e.g. to insert latitude and longitude from a filter, like in the above example.
 
 .. _sort:
@@ -215,4 +248,14 @@ Example::
         mode = filter
     }
 
-Only ``filter`` is allowed as value. Will submit an empty query to switch to filter mode.
+Only ``filter`` is allowed as value, as ``search`` is default behaviour. Using ``filter`` will
+trigger a search to provide data while visiting the page, possible :ref:`filter` allow you to build
+pages like "News".
+
+.. _searching_dataprocessing:
+
+dataProcessing
+--------------
+
+Configure modifications on each document before returning search result.
+For full documentation check out :ref:`dataprocessors`.

Documentation/source/connections.rst

@@ -5,28 +5,25 @@ Connections
 
 See Concept of :ref:`concepts_connections` for further background information.
 
-The extension provides the following connections out of the box:
+For information about implementing a new connection, take a look at :ref:`development_connection`.
 
-.. _Elasticsearch:
+The following connections were developed, or are in development, for ``search_core``:
+
+.. _connection_elasticsearch:
 
 Elasticsearch
 -------------
 
-Integrates `elastic Elasticsearch`_ using `elastica`_ into TYPO3.
+Support for `Elasticsearch`_ is provided out of the box by `search_core` at the moment.
 
-Provides basic support like indexing without mappings and full text search at the moment.
+.. _Elasticsearch: https://www.elastic.co/products/elasticsearch
 
-The connection is configurable through the following options:
+.. _connection_algolia:
 
-* :ref:`host`
+Algolia
+-------
 
-* :ref:`port`
+`search_algolia`_ will integrate `Algolia`_ and is currently under development by Martin Hummer.
 
-* :ref:`index`
-
-* :ref:`mapping`
-
-* :ref:`facets`
-
-.. _elastic Elasticsearch: https://www.elastic.co/products/elasticsearch
-.. _elastica: http://elastica.io/
+.. _search_algolia: https://github.com/martinhummer/search_algolia/
+.. _Algolia: https://www.algolia.com/

Documentation/source/dataprocessors.rst

@@ -0,0 +1,84 @@
+.. _dataprocessors:
+
+DataProcessors
+==============
+
+See Concept of :ref:`concepts_dataprocessing` for further background information.
+
+For information about implementing a new DataProcessor, take a look at
+:ref:`development_dataprocessor`.
+
+Same as provided by TYPO3 for :ref:`t3tsref:cobj-fluidtemplate` through
+:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`.
+
+.. _dataprocessors_usage:
+
+Usage
+-----
+
+All processors are applied in configured order. Allowing to work with already processed data.
+They can be applied during indexing and for search results.
+
+Example for indexing::
+
+    plugin.tx_searchcore.settings.indexing.pages.dataProcessing {
+        1 = Codappix\SearchCore\DataProcessing\CopyToProcessor
+        1 {
+            to = search_spellcheck
+        }
+
+        2 = Codappix\SearchCore\DataProcessing\CopyToProcessor
+        2 {
+            to = search_all
+        }
+    }
+
+The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards
+all fields, including ``search_spellcheck`` will be copied to ``search_all``.
+
+Example for search results::
+
+    plugin.tx_searchcore.settings.searching.dataProcessing {
+        1 = Codappix\SearchCore\DataProcessing\CopyToProcessor
+        1 {
+            to = search_spellcheck
+        }
+
+        2 = Codappix\SearchCore\DataProcessing\CopyToProcessor
+        2 {
+            to = search_all
+        }
+    }
+
+The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards
+all fields, including ``search_spellcheck`` will be copied to ``search_all``.
+
+.. _dataprocessors_availableDataProcessors:
+
+Available DataProcessors
+------------------------
+
+.. toctree::
+    :maxdepth: 1
+    :glob:
+
+    /configuration/dataProcessing/ContentObjectDataProcessorAdapterProcessor
+    /configuration/dataProcessing/CopyToProcessor
+    /configuration/dataProcessing/GeoPointProcessor
+    /configuration/dataProcessing/RemoveProcessor
+    /configuration/dataProcessing/TcaRelationResolvingProcessor
+
+.. _dataprocessors_plannedDataProcessors:
+
+Planned DataProcessors
+----------------------
+
+    ``Codappix\SearchCore\DataProcessing\ReplaceProcessor``
+        Will execute a search and replace on configured fields.
+
+    ``Codappix\SearchCore\DataProcessing\RootLevelProcessor``
+        Will attach the root level to the record.
+
+    ``Codappix\SearchCore\DataProcessing\ChannelProcessor``
+        Will add a configurable channel to the record, e.g. if you have different areas in your
+        website like "products" and "infos".

Documentation/source/development.rst

@@ -1,71 +1,16 @@
 .. highlight:: bash
 
-.. _contribution:
-
-Contribution
-============
-
-Everyone is welcome to contribute, whether it's code, issues, feature requests or any other kind.
-
-Below is a documentation what to respect during contributions.
-
-.. _contribution_setup:
-
-Setup
------
-
-To start contributions regarding code, make sure your environment matches the following
-requirements:
-
-* composer is executable
-
-* PHP on CLI is executable
-
-* MySQL is up and running with user *dev* and password *dev* on *127.0.0.1* or to overwrite the
-  environment variables, see :file:`Makefile`.
-  And MySQL is not set to strict mode as TYPO3 doesn't support strict mode, see
-  https://review.typo3.org/#/c/26725/3/INSTALL.md.
-
-* Elasticsearch is installed and up and running on *localhost:9200*.
-
-Then setup your system::
-
-    git clone git@github.com:DanielSiepmann/search_core.git \
-        && cd search_core \
-        && export typo3DatabaseName="searchcoretest76" \
-        && export TYPO3_VERSION="~7.6" \
-        && make install \
-        && make unitTests \
-        && make functionalTests
-
-If all tests are okay, start your work.
-
-If you are working with multiple TYPO3 versions make sure to export `typo3DatabaseName` and
-`TYPO3_VERSION` in your environment like::
-
-    export typo3DatabaseName="searchcoretest62"
-    export TYPO3_VERSION="~6.2"
-
-Also run the install command for each version before running any tests. Only this will make sure you
-are testing against the actual TYPO3 Version and database scheme.
-
-.. _contribution_development:
-
 Development
------------
+===========
 
-All changes are introduced through pull requests at `Github`_ and should contain the following:
+There are some ways we will cover here. One is how you can develop own parts like Indexer,
+DataProcessor and Connection. The other is how to contribute.
 
-* Adjusted tests if tests existed before. Otherwise they will break on `travis-ci`_.
+.. toctree::
+   :maxdepth: 1
+   :glob:
 
-* New tests whenever possible and useful.
-
-* Code has to follow `PSR-2`_.
-
-* Adjusted documentation.
-
-* Make sure to follow the documented :ref:`concepts`.
-
-.. _Github: https://github.com/DanielSiepmann/search_core
-.. _travis-ci: https://travis-ci.org/
-.. _PSR-2: http://www.php-fig.org/psr/psr-2/
+   development/indexer
+   development/dataProcessor
+   development/connection
+   development/contribution

Documentation/source/development/connection.rst

@@ -0,0 +1,11 @@
+.. _development_connection:
+
+Develop a new Connection
+========================
+
+Make sure you understood :ref:`concepts_connections`.
+
+Each Connection has to be a single class which implements
+``Codappix\SearchCore\Connection\ConnectionInterface``.
+
+Dependency Injection is working for custom Connections.

Documentation/source/development/contribution.rst

@@ -0,0 +1,69 @@
+.. _contribution:
+
+Contribution
+============
+
+Everyone is welcome to contribute, whether it's code, issues, feature requests or any other kind.
+
+Below is a documentation what to respect during contributions.
+
+.. _contribution_setup:
+
+Setup
+-----
+
+To start contributions regarding code, make sure your environment matches the following
+requirements:
+
+* composer is executable
+
+* PHP on CLI is executable
+
+* MySQL is up and running with user *dev* and password *dev* on *127.0.0.1* or to overwrite the
+  environment variables, see :file:`Makefile`.
+  And MySQL is not set to strict mode as TYPO3 doesn't support strict mode, see
+  https://review.typo3.org/#/c/26725/3/INSTALL.md.
+
+* Elasticsearch is installed and up and running on *localhost:9200*.
+
+Then setup your system::
+
+    git clone git@github.com:codappix/search_core.git \
+        && cd search_core \
+        && export typo3DatabaseName="searchcoretest87" \
+        && export TYPO3_VERSION="~8.7" \
+        && make install \
+        && make unitTests \
+        && make functionalTests
+
+If all tests are okay, start your work.
+
+If you are working with multiple TYPO3 versions make sure to export `typo3DatabaseName` and
+`TYPO3_VERSION` in your environment like::
+
+    export typo3DatabaseName="searchcoretest76"
+    export TYPO3_VERSION="~7.6"
+
+Also run the install command for each version before running any tests. Only this will make sure you
+are testing against the actual TYPO3 Version and database scheme.
+
+.. _contribution_development:
+
+Development
+-----------
+
+All changes are introduced through pull requests at `Github`_ and should contain the following:
+
+* Adjusted tests if tests existed before. Otherwise they will break on `travis-ci`_.
+
+* New tests whenever possible and useful.
+
+* Code has to follow `PSR-2`_.
+
+* Adjusted documentation.
+
+* Make sure to follow the documented :ref:`concepts`.
+
+.. _Github: https://github.com/codappix/search_core
+.. _travis-ci: https://travis-ci.org/
+.. _PSR-2: http://www.php-fig.org/psr/psr-2/

Documentation/source/development/dataProcessor.rst

@@ -0,0 +1,15 @@
+.. _development_dataprocessor:
+
+Develop a new DataProcessor
+===========================
+
+Make sure you understood :ref:`concepts_dataprocessing`.
+
+Each DataProcessor has to be a single class which implements
+``Codappix\SearchCore\DataProcessing\ProcessorInterface``.
+
+Make sure you support both, Frontend and Backend, as processors can be called during searching and
+indexing. Therefore do not rely on e.g. ``TSFE``, make sure dependencies are met and your code will
+work in both environments.
+
+Dependency Injection is working for custom DataProcessors.

Documentation/source/development/indexer.rst

@@ -0,0 +1,21 @@
+.. _development_indexer:
+
+Develop a new Indexer
+=====================
+
+Make sure you understood :ref:`concepts_indexing`.
+
+Each indexer has to be a single class which implements
+``Codappix\SearchCore\Domain\Index\IndexerInterface``.
+
+The indexer should call the connection with all necessary information about the document(s) to
+trigger indexing or deletion of whole index.
+
+As this is the "indexer", deletion of single documents is directly processed by the connection.
+
+``setIdentifier`` is called with the identifier of the current Indexer. This might be useful to
+fetch configuration, related to the indexing, from
+``Codappix\SearchCore\Configuration\ConfigurationContainerInterface``.
+
+Dependency Injection is working for custom indexers, therefore you are able to inject the
+``ConfigurationContainerInterface``.

Documentation/source/features.rst

@@ -3,38 +3,60 @@
 Features
 ========
 
-The following features are currently provided:
+The following features are available:
 
 .. _features_indexing:
 
 Indexing
 --------
 
-Indexing data to Elasticsearch is provided. The extension delivers an indexer for TCA with zero
-configuration needs. Still it's possible to configure the indexer.
+Indexing of data is possible. We deliver an indexer for TCA with zero configuration needs. You can
+also provide custom indexer for arbitrary data which is not indexable through TCA.
 
-Also custom classes can be used as indexers.
+Also a finisher for TYPO3 Form-Extension is provided to integrate indexing after something was
+update through the Form-Extension.
 
-Furthermore a finisher for TYPO3 Form-Extension is provided to integrate indexing.
+Indexing is done through Hooks and CLI. We therefore provide commands to index and delete indexed
+data.
 
 .. _features_search:
 
 Searching
 ---------
 
-Currently all fields are searched for a single search input.
+.. note::
+    Currently only integrated for Elasticsearch with no abstraction.
+    If you need to implement your own search, please open an issue on Github and we will change the code
+    base.
 
-Also multiple filter are supported. Filtering results by fields for string contents.
+Via TypoScript it's possible to configure the fields to query, minimum match and script fields.
+Also multiple filter are supported, filtering results by fields.
 
 Facets / aggregates are also possible. Therefore a mapping has to be defined in TypoScript for
 indexing, and the facets itself while searching.
 
+.. _features_dataProcessing:
+
+DataProcessing
+--------------
+
+DataProcessing, as known from ``FLUIDTEMPLATE``, is available while indexing and for search results.
+Each record and result item can be processed by multiple processor to prepare data for indexing and
+output.
+
+See :ref:`concepts_dataprocessing` in :ref:`concepts` section.
+
 .. _features_planned:
 
 Planned
----------
+-------
 
 The following features are currently planned and will be integrated:
 
-#. Pagination
-   Add a pagination to search results, to allow users to walk through all results.
+#. :issue:`25` Multi language.
+#. :issue:`94` Respect access rights while indexing relations.
+#. :issue:`75` Configuration of index name (for Elasticsearch).
+
+For a full list, check out our `open issues`_.
+
+.. _open issues: https://github.com/Codappix/search_core/issues

Documentation/source/index.rst

@@ -1,7 +1,7 @@
 .. include:: readme.rst
 
 Table of Contents
-=================
+-----------------
 
 .. toctree::
    :maxdepth: 1
@@ -14,4 +14,6 @@ Table of Contents
    concepts
    connections
    indexer
+   dataprocessors
    development
+   changelog

Documentation/source/indexer.rst

@@ -5,6 +5,8 @@ Indexer
 
 See Concept of :ref:`concepts_indexing` for further background information.
 
+For information about implementing a new indexer, take a look at :ref:`development_indexer`.
+
 The extension provides the following indexer out of the box:
 
 .. _TcaIndexer:
@@ -15,23 +17,12 @@ TcaIndexer
 Provides zero configuration TYPO3 integration by using the :ref:`t3tcaref:start`. You just can
 start indexing TYPO3.
 
-The indexer will use the TCA to fetch all necessary information like relations. Currently the
-implementation is very basic. In future it will also provide mapping for :ref:`Elasticsearch` and
-further stuff.
+Just add the indexer for a TYPO3 table. The indexer will use the TCA to fetch all necessary
+information like relations.
 
-The indexer is configurable through the following options:
+.. note::
 
-* :ref:`rootLineBlacklist`
-
-* :ref:`additionalWhereClause`
-
-* :ref:`abstractFields`
-
-* :ref:`mapping`
-
-* :ref:`index`
-
-* :ref:`dataProcessing`
+  Not all relations are resolved yet, see :issue:`17`.
 
 .. _PagesIndexer:
 
@@ -42,23 +33,7 @@ Provides zero configuration TYPO3 integration by using the :ref:`t3tcaref:start`
 start indexing TYPO3.
 
 The indexer will use the TCA to fetch all necessary information like relations. Currently the
-implementation is very basic. In future it will also provide mapping for :ref:`Elasticsearch` and
-further stuff. Also all static content from each page will be concatenated into a single field to
-improve search.
-
-The indexer is configurable through the following options:
-
-* :ref:`rootLineBlacklist`
-
-* :ref:`additionalWhereClause`
-
-* :ref:`abstractFields`
-
-* :ref:`mapping`
-
-* :ref:`index`
-
-* :ref:`dataProcessing`
+implementation is very basic.
 
 .. note::
 

Documentation/source/installation.rst

@@ -4,19 +4,41 @@
 Installation
 ============
 
+Composer
+--------
+
 The extension can be installed through composer::
 
-    composer require "leonmrni/search_core dev-master as 1.0.x-dev"
+    composer require "codappix/search_core" "~0.0.4"
 
-or by `downloading`_ and placing it inside the :file:`typo3conf/ext`-Folder of your installation.
-In that case you need to install all dependencies yourself. Dependencies are:
+Note that you have to allow unstable packages:
+
+.. code-block:: json
+
+   {
+      "minimum-stability": "dev",
+      "prefer-stable": true
+   }
+
+Download
+--------
+
+You can also `download`_ the extension and placing it inside the :file:`typo3conf/ext`-Folder of
+your installation.  In that case you need to install all dependencies yourself. Dependencies are:
 
 .. literalinclude:: ../../composer.json
    :caption: Dependencies from composer.json
    :lines: 19-21
    :dedent: 8
 
+Setup
+-----
+
 Afterwards you need to enable the extension through the extension manager and include the static
 TypoScript setup.
 
-.. _downloading: https://github.com/DanielSiepmann/search_core/archive/master.zip
+If you **don't** want to use the included Elasticsearch integration, you have to disable it in the
+extension manager configuration of the extension by checking the checkbox.
+It's currently enabled by default but will be moved into its own extension in the future.
+
+.. _download: https://github.com/codappix/search_core/archive/develop.zip

Documentation/source/readme.rst

@@ -1,5 +1,5 @@
-TYPO3 Extension search_core's documentation!
-============================================
+TYPO3 Extension search_core
+===========================
 
 Introduction
 ============
@@ -8,24 +8,21 @@ What does it do?
 ----------------
 
 The goal of this extension is to provide search integrations into TYPO3 CMS. The extension will
-abstract the concrete implementations to allow exchange of concrete backends like Elasticsearch or
-solr.
+provide a convenient API to allow developers to provide concrete implementations of backends like
+Elasticsearch, Algolia or Solr.
 
 The extension provides integration into TYPO3 like a frontend plugin for searches and hooks to
 update search indexes on updates. Also a command line interface is provided for interactions like
-reindexing.
+re-indexing.
 
 Current state
 -------------
 
 This is still a very early beta version. More information can be taken from Github at
-`current issues`_ and `current projects`_.
+`current issues`_.
 
-We are also focusing on Code Quality and Testing through `travis ci`_, `scrutinizer`_ and `codacy`_.
+We are also focusing on Code Quality and Testing through `travis ci`_, ``phpcs``, ``phpunit`` and
+``phpstan``.
 
 .. _current issues: https://github.com/Codappix/search_core/issues
-.. _current projects: https://github.com/Codappix/search_core/projects
 .. _travis ci: https://travis-ci.org/Codappix/search_core
-.. _scrutinizer: https://scrutinizer-ci.com/g/Codappix/search_core/inspections
-.. _codacy: https://www.codacy.com/app/Codappix/search_core/dashboard
-

Documentation/source/usage.rst

@@ -11,9 +11,10 @@ Manual indexing
 
 You can trigger indexing from CLI::
 
-    ./typo3/cli_dispatch.phpsh extbase index:index --identifier 'tt_content'
+    ./typo3/cli_dispatch.phpsh extbase index:index --identifier 'pages'
+    ./bin/typo3cms index:index --identifier 'pages'
 
-This will index the table ``tt_content`` using the :ref:`TcaIndexer`.
+This will index the table ``pages`` using the :ref:`TcaIndexer`.
 
 Only one index per call is available, to run multiple indexers, just make multiple calls.
 The indexers have to be defined in TypoScript via :ref:`configuration_options_index`.
@@ -25,9 +26,10 @@ Manual deletion
 
 You can trigger deletion for a single index from CLI::
 
-    ./typo3/cli_dispatch.phpsh extbase index:delete --identifier 'tt_content'
+    ./typo3/cli_dispatch.phpsh extbase index:delete --identifier 'pages'
+    ./bin/typo3cms index:delete --identifier 'pages'
 
-This will delete the index for the table ``tt_content``.
+This will delete the index for the table ``pages``.
 
 Only one delete per call is available, to run multiple deletions, just make multiple calls.
 
@@ -53,7 +55,6 @@ A form finisher is provided to integrate indexing into form extension.
 Add form finisher to your available finishers and configure it like:
 
 .. code-block:: yaml
-   :linenos:
 
     -
         identifier: SearchCoreIndexer
@@ -62,7 +63,7 @@ Add form finisher to your available finishers and configure it like:
             indexIdentifier: 'fe_users'
             recordUid: '{FeUser.user.uid}'
 
-All three options are necessary, where
+All three options are necessary, where:
 
 ``action``
     Is one of ``delete``, ``update`` or ``add``.
@@ -81,7 +82,7 @@ plugin. The plugin is named *Search Core*.
 
 Please provide your own template, the extension will not deliver a useful template for now.
 
-The extbase mapping is used, this way you can create a form:
+The Extbase mapping is used, this way you can create a form:
 
 .. code-block:: html
 
@@ -95,11 +96,10 @@ The extbase mapping is used, this way you can create a form:
 Filter
 """"""
 
-Thanks to extbase mapping, filter are added to the form:
+Thanks to Extbase mapping, filter are added to the form:
 
 .. code-block:: html
 
-   <!-- Case sensitive for fields of type keyword. -->
    <f:form.textfield property="filter.exampleName" value="the value to match" />
 
 .. _usage_searching_facets:

Makefile

@@ -5,16 +5,28 @@ TYPO3_WEB_DIR := $(current_dir).Build/web
 TYPO3_PATH_ROOT := $(current_dir).Build/web
 # Allow different versions on travis
 TYPO3_VERSION ?= ~8.7
-typo3DatabaseName ?= "searchcore_test2"
+typo3DatabaseName ?= "searchcore_test"
 typo3DatabaseUsername ?= "dev"
 typo3DatabasePassword ?= "dev"
 typo3DatabaseHost ?= "127.0.0.1"
 
+sourceOrDist=--prefer-dist
+ifeq ($(TYPO3_VERSION),~7.6)
+	sourceOrDist=--prefer-source
+endif
+
 .PHONY: install
 install: clean
-	COMPOSER_PROCESS_TIMEOUT=1000 composer require -vv --dev --prefer-dist --ignore-platform-reqs typo3/cms="$(TYPO3_VERSION)"
+	if [ $(TYPO3_VERSION) = ~7.6 ]; then \
+		patch composer.json Tests/InstallPatches/composer.json.patch; \
+	fi
+
+	COMPOSER_PROCESS_TIMEOUT=1000 composer require -vv --dev $(sourceOrDist) typo3/cms="$(TYPO3_VERSION)"
 	git checkout composer.json
 
+cgl:
+	./.Build/bin/phpcs
+
 functionalTests:
 	typo3DatabaseName=$(typo3DatabaseName) \
 		typo3DatabaseUsername=$(typo3DatabaseUsername) \
@@ -29,16 +41,11 @@ unitTests:
 		.Build/bin/phpunit --colors --debug -v \
 		-c Tests/Unit/UnitTests.xml
 
-uploadCodeCoverage: uploadCodeCoverageToScrutinizer uploadCodeCoverageToCodacy
+uploadCodeCoverage: uploadCodeCoverageToScrutinizer
 
 uploadCodeCoverageToScrutinizer:
 	wget https://scrutinizer-ci.com/ocular.phar && \
 	php ocular.phar code-coverage:upload --format=php-clover .Build/report/functional/clover/coverage
 
-uploadCodeCoverageToCodacy:
-	composer require -vv --dev codacy/coverage && \
-	git checkout composer.json && \
-	php .Build/bin/codacycoverage clover .Build/report/functional/clover/coverage
-
 clean:
 	rm -rf .Build composer.lock

Tests/Functional/AbstractFunctionalTestCase.php

@@ -66,4 +66,9 @@ abstract class AbstractFunctionalTestCase extends CoreTestCase
     {
         return ['EXT:search_core/Tests/Functional/Fixtures/BasicSetup.ts'];
     }
+
+    protected function isLegacyVersion() : bool
+    {
+        return \TYPO3\CMS\Core\Utility\VersionNumberUtility::convertVersionNumberToInteger(TYPO3_version) < 8000000;
+    }
 }

Tests/Functional/Bootstrap.php

@@ -0,0 +1,9 @@
+<?php
+
+$filePath = '.Build/vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTestsBootstrap.php';
+
+if (getenv('TYPO3_VERSION') === '~7.6') {
+    $filePath = '.Build/vendor/typo3/cms/typo3/sysext/core/Build/FunctionalTestsBootstrap.php';
+}
+
+require_once dirname(dirname(__DIR__)) . '/' . $filePath;

Tests/Functional/Connection/Elasticsearch/FacetTest.php

@@ -0,0 +1,78 @@
+<?php
+namespace Codappix\SearchCore\Tests\Functional\Connection\Elasticsearch;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\SearchCore\Domain\Index\IndexerFactory;
+use Codappix\SearchCore\Domain\Model\SearchRequest;
+use Codappix\SearchCore\Domain\Search\SearchService;
+use TYPO3\CMS\Extbase\Object\ObjectManager;
+
+class FacetTest extends AbstractFunctionalTestCase
+{
+    protected function getTypoScriptFilesForFrontendRootPage()
+    {
+        return array_merge(
+            parent::getTypoScriptFilesForFrontendRootPage(),
+            ['EXT:search_core/Tests/Functional/Fixtures/Searching/Facet.ts']
+        );
+    }
+
+    protected function getDataSets()
+    {
+        return array_merge(
+            parent::getDataSets(),
+            ['Tests/Functional/Fixtures/Searching/Filter.xml']
+        );
+    }
+
+    /**
+     * @test
+     */
+    public function itsPossibleToFetchFacetsForField()
+    {
+        \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class)
+            ->get(IndexerFactory::class)
+            ->getIndexer('tt_content')
+            ->indexAllDocuments()
+            ;
+
+        $searchService = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class)
+            ->get(SearchService::class);
+
+        $searchRequest = new SearchRequest();
+        $result = $searchService->search($searchRequest);
+
+        $this->assertSame(1, count($result->getFacets()), 'Did not receive the single defined facet.');
+
+        $facet = current($result->getFacets());
+        $this->assertSame('contentTypes', $facet->getName(), 'Name of facet was not as expected.');
+        $this->assertSame('CType', $facet->getField(), 'Field of facet was not expected.');
+
+        $options = $facet->getOptions();
+        $this->assertSame(2, count($options), 'Did not receive the expected number of possible options for facet.');
+        $option = $options['HTML'];
+        $this->assertSame('HTML', $option->getName(), 'Option did not have expected Name.');
+        $this->assertSame(1, $option->getCount(), 'Option did not have expected count.');
+        $option = $options['Header'];
+        $this->assertSame('Header', $option->getName(), 'Option did not have expected Name.');
+        $this->assertSame(1, $option->getCount(), 'Option did not have expected count.');
+    }
+}

Tests/Functional/Connection/Elasticsearch/FilterTest.php

@@ -55,40 +55,7 @@ class FilterTest extends AbstractFunctionalTestCase
 
         $searchRequest->setFilter(['CType' => 'HTML']);
         $result = $searchService->search($searchRequest);
-        $this->assertSame(5, $result->getResults()[0]['uid'], 'Did not get the expected result entry.');
+        $this->assertSame(5, (int) $result->getResults()[0]['uid'], 'Did not get the expected result entry.');
         $this->assertSame(1, count($result), 'Did not receive the single filtered element.');
     }
-
-    /**
-     * @test
-     */
-    public function itsPossibleToFetchFacetsForField()
-    {
-        \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class)
-            ->get(IndexerFactory::class)
-            ->getIndexer('tt_content')
-            ->indexAllDocuments()
-            ;
-
-        $searchService = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class)
-            ->get(SearchService::class);
-
-        $searchRequest = new SearchRequest('Search Word');
-        $result = $searchService->search($searchRequest);
-
-        $this->assertSame(1, count($result->getFacets()), 'Did not receive the single defined facet.');
-
-        $facet = current($result->getFacets());
-        $this->assertSame('contentTypes', $facet->getName(), 'Name of facet was not as expected.');
-        $this->assertSame('CType', $facet->getField(), 'Field of facet was not expected.');
-
-        $options = $facet->getOptions();
-        $this->assertSame(2, count($options), 'Did not receive the expected number of possible options for facet.');
-        $option = $options['HTML'];
-        $this->assertSame('HTML', $option->getName(), 'Option did not have expected Name.');
-        $this->assertSame(1, $option->getCount(), 'Option did not have expected count.');
-        $option = $options['Header'];
-        $this->assertSame('Header', $option->getName(), 'Option did not have expected Name.');
-        $this->assertSame(1, $option->getCount(), 'Option did not have expected count.');
-    }
 }

Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php

@@ -49,12 +49,11 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
 
         $response = $this->client->request('typo3content/_search?q=*:*');
 
-        $this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.');
-        $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.');
-        $this->assertArraySubset(
-            ['_source' => ['header' => 'indexed content element']],
-            $response->getData()['hits']['hits'][1],
-            false,
+        $this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.');
+        $this->assertSame($response->getData()['hits']['total'], 3, 'Not exactly 3 documents were indexed.');
+        $this->assertSame(
+            'indexed content element',
+            $response->getData()['hits']['hits'][2]['_source']['header'],
             'Record was not indexed.'
         );
     }
@@ -72,7 +71,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
 
         $response = $this->client->request('typo3content/_search?q=*:*');
 
-        $this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.');
+        $this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.');
         $this->assertSame($response->getData()['hits']['total'], 1, 'Not exactly 1 document was indexed.');
         $this->assertArraySubset(
             ['_source' => ['header' => 'indexed content element']],
@@ -112,8 +111,8 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
 
         $response = $this->client->request('typo3content/_search?q=*:*');
 
-        $this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.');
-        $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.');
+        $this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.');
+        $this->assertSame($response->getData()['hits']['total'], 3, 'Not exactly 3 documents were indexed.');
     }
 
     /**
@@ -135,8 +134,8 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
 
         $response = $this->client->request('typo3content/_search?q=*:*');
 
-        $this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.');
-        $this->assertSame($response->getData()['hits']['total'], 3, 'Not exactly 3 documents were indexed.');
+        $this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.');
+        $this->assertSame($response->getData()['hits']['total'], 4, 'Not exactly 4 documents were indexed.');
         $response = $this->client->request('typo3content/_search?q=uid:11');
         $this->assertArraySubset(
             ['_source' => ['header' => 'Also indexable record']],
@@ -167,8 +166,8 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
             ;
 
         $response = $this->client->request('typo3content/_search?q=*:*');
-        $this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.');
-        $this->assertSame($response->getData()['hits']['total'], 4, 'Not exactly 4 documents were indexed.');
+        $this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.');
+        $this->assertSame($response->getData()['hits']['total'], 5, 'Not exactly 5 documents were indexed.');
 
         $response = $this->client->request('typo3content/_search?q=uid:11');
         $this->assertArraySubset(
@@ -209,7 +208,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
     /**
      * @test
      */
-    public function indexingDeltedRecordIfRecordShouldBeIndexedButIsNoLongerAvailableAndWasAlreadyIndexed()
+    public function indexingDeletedRecordIfRecordShouldBeIndexedButIsNoLongerAvailableAndWasAlreadyIndexed()
     {
         \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class)
             ->get(IndexerFactory::class)
@@ -218,14 +217,19 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
             ;
 
         $response = $this->client->request('typo3content/_search?q=*:*');
-        $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.');
+        $this->assertSame($response->getData()['hits']['total'], 3, 'Not exactly 3 documents were indexed.');
 
-        $this->getConnectionPool()->getConnectionForTable('tt_content')
-            ->update(
-                'tt_content',
-                ['hidden' => true],
-                ['uid' => 10]
-            );
+        if ($this->isLegacyVersion()) {
+            $this->getDatabaseConnection()
+                ->exec_UPDATEquery('tt_content', 'uid = 10', ['hidden' => 1]);
+        } else {
+            $this->getConnectionPool()->getConnectionForTable('tt_content')
+                ->update(
+                    'tt_content',
+                    ['hidden' => true],
+                    ['uid' => 10]
+                );
+        }
 
         \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class)
             ->get(IndexerFactory::class)
@@ -234,6 +238,6 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
             ;
 
         $response = $this->client->request('typo3content/_search?q=*:*');
-        $this->assertSame($response->getData()['hits']['total'], 1, 'Not exactly 1 document is in index.');
+        $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 document is in index.');
     }
 }

Tests/Functional/DataProcessing/ContentObjectDataProcessorAdapterProcessorTest.php

@@ -0,0 +1,62 @@
+<?php
+namespace Codappix\SearchCore\Tests\Functional\DataProcessing;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\SearchCore\Compatibility\TypoScriptService76;
+use Codappix\SearchCore\Compatibility\TypoScriptService;
+use Codappix\SearchCore\DataProcessing\ContentObjectDataProcessorAdapterProcessor;
+use Codappix\SearchCore\Tests\Functional\AbstractFunctionalTestCase;
+use TYPO3\CMS\Frontend\DataProcessing\SplitProcessor;
+
+class ContentObjectDataProcessorAdapterProcessorTest extends AbstractFunctionalTestCase
+{
+    /**
+     * @test
+     */
+    public function contentObjectDataProcessorIsExecuted()
+    {
+        $record = ['content' => 'value1, value2'];
+        $configuration = [
+            '_dataProcessor' => SplitProcessor::class,
+            'delimiter' => ',',
+            'fieldName' => 'content',
+            'as' => 'new_content',
+        ];
+        $expectedData = [
+            'content' => 'value1, value2',
+            'new_content' => ['value1', 'value2'],
+        ];
+
+        if ($this->isLegacyVersion()) {
+            $typoScriptService = new TypoScriptService76();
+        } else {
+            $typoScriptService = new TypoScriptService();
+        }
+
+        $subject = new ContentObjectDataProcessorAdapterProcessor($typoScriptService);
+        $processedData = $subject->processData($record, $configuration);
+        $this->assertSame(
+            $expectedData,
+            $processedData,
+            'The processor did not return the expected processed record.'
+        );
+    }
+}

Tests/Functional/DataProcessing/TcaRelationResolvingProcessorTest.php

@@ -1,5 +1,5 @@
 <?php
-namespace Codappix\SearchCore\Tests\Indexing\TcaIndexer;
+namespace Codappix\SearchCore\Tests\Functional\DataProcessing;
 
 /*
  * Copyright (C) 2016  Daniel Siepmann <coding@daniel-siepmann.de>
@@ -20,13 +20,13 @@ namespace Codappix\SearchCore\Tests\Indexing\TcaIndexer;
  * 02110-1301, USA.
  */
 
-use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService;
+use Codappix\SearchCore\DataProcessing\TcaRelationResolvingProcessor;
 use Codappix\SearchCore\Tests\Functional\AbstractFunctionalTestCase;
 use TYPO3\CMS\Backend\Utility\BackendUtility;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 use TYPO3\CMS\Extbase\Object\ObjectManager;
 
-class RelationResolverTest extends AbstractFunctionalTestCase
+class TcaRelationResolvingProcessorTest extends AbstractFunctionalTestCase
 {
     /**
      * @test
@@ -37,9 +37,8 @@ class RelationResolverTest extends AbstractFunctionalTestCase
         $objectManager = GeneralUtility::makeInstance(ObjectManager::class);
         $table = 'sys_file';
 
-        $subject = $objectManager->get(TcaTableService::class, $table);
-        $record = BackendUtility::getRecord($table, 1);
-        $subject->prepareRecord($record);
+        $subject = $objectManager->get(TcaRelationResolvingProcessor::class);
+        $record = $subject->processData(BackendUtility::getRecord($table, 1), ['_table' => $table]);
 
         $this->assertEquals(
             [
@@ -60,9 +59,8 @@ class RelationResolverTest extends AbstractFunctionalTestCase
         $objectManager = GeneralUtility::makeInstance(ObjectManager::class);
         $table = 'tt_content';
 
-        $subject = $objectManager->get(TcaTableService::class, $table);
-        $record = BackendUtility::getRecord($table, 1);
-        $subject->prepareRecord($record);
+        $subject = $objectManager->get(TcaRelationResolvingProcessor::class);
+        $record = $subject->processData(BackendUtility::getRecord($table, 1), ['_table' => $table]);
 
         $this->assertEquals(
             'Insert Plugin',
@@ -80,9 +78,8 @@ class RelationResolverTest extends AbstractFunctionalTestCase
         $objectManager = GeneralUtility::makeInstance(ObjectManager::class);
         $table = 'tt_content';
 
-        $subject = $objectManager->get(TcaTableService::class, $table);
-        $record = BackendUtility::getRecord($table, 1);
-        $subject->prepareRecord($record);
+        $subject = $objectManager->get(TcaRelationResolvingProcessor::class);
+        $record = $subject->processData(BackendUtility::getRecord($table, 1), ['_table' => $table]);
 
         $this->assertEquals(
             [
@@ -103,9 +100,8 @@ class RelationResolverTest extends AbstractFunctionalTestCase
         $objectManager = GeneralUtility::makeInstance(ObjectManager::class);
         $table = 'tt_content';
 
-        $subject = $objectManager->get(TcaTableService::class, $table);
-        $record = BackendUtility::getRecord($table, 1);
-        $subject->prepareRecord($record);
+        $subject = $objectManager->get(TcaRelationResolvingProcessor::class);
+        $record = $subject->processData(BackendUtility::getRecord($table, 1), ['_table' => $table]);
 
         $this->assertEquals(
             [

Tests/Functional/Fixtures/BasicSetup.ts

@@ -15,7 +15,7 @@ plugin {
 
                     additionalWhereClause (
                         tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu', 'shortcut', 'search', 'login')
-                        AND tt_content.bodytext != ''
+                        AND (tt_content.bodytext != '' OR tt_content.header != '')
                     )
 
                     mapping {
@@ -23,27 +23,30 @@ plugin {
                             type = keyword
                         }
                     }
+
+                    dataProcessing {
+                        1 = Codappix\SearchCore\DataProcessing\TcaRelationResolvingProcessor
+                    }
                 }
 
                 pages {
                     indexer = Codappix\SearchCore\Domain\Index\TcaIndexer\PagesIndexer
                     abstractFields = abstract, description, bodytext
+                    contentFields = header, bodytext
 
                     mapping {
                         CType {
                             type = keyword
                         }
                     }
+
+                    dataProcessing {
+                        1 = Codappix\SearchCore\DataProcessing\TcaRelationResolvingProcessor
+                    }
                 }
             }
 
             searching {
-                facets {
-                    contentTypes {
-                        field = CType
-                    }
-                }
-
                 fields {
                     query = _all
                 }

Tests/Functional/Fixtures/Indexing/IndexTcaTable.xml

@@ -99,4 +99,31 @@
         <colPos>0</colPos>
         <filelink_sorting>0</filelink_sorting>
     </tt_content>
+
+    <tt_content>
+        <uid>100</uid>
+        <pid>2</pid>
+        <tstamp>1480686370</tstamp>
+        <crdate>1480686370</crdate>
+        <hidden>0</hidden>
+        <sorting>72</sorting>
+        <CType>header</CType>
+        <header>Indexed on page 2</header>
+        <bodytext>This element is on a different page</bodytext>
+        <media>0</media>
+        <layout>0</layout>
+        <deleted>0</deleted>
+        <cols>0</cols>
+        <starttime>0</starttime>
+        <endtime>0</endtime>
+        <colPos>0</colPos>
+        <filelink_sorting>0</filelink_sorting>
+    </tt_content>
+
+    <pages>
+        <uid>2</uid>
+        <pid>1</pid>
+        <title>Second page with content</title>
+        <description>Used to check whether content is indexed only for parent page.</description>
+    </pages>
 </dataset>

Tests/Functional/Fixtures/Indexing/TcaIndexer/KeepSysLanguageUid.xml

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="utf-8"?>
+<dataset>
+    <tt_content>
+        <uid>1</uid>
+        <pid>1</pid>
+        <tstamp>1480686370</tstamp>
+        <crdate>1480686370</crdate>
+        <hidden>0</hidden>
+        <sorting>72</sorting>
+        <sys_language_uid>2</sys_language_uid>
+        <CType>header</CType>
+        <header>indexed content element</header>
+        <bodytext>this is the content of header content element that should get indexed</bodytext>
+        <media>0</media>
+        <layout>0</layout>
+        <deleted>0</deleted>
+        <cols>0</cols>
+        <starttime>0</starttime>
+        <endtime>0</endtime>
+        <colPos>0</colPos>
+        <filelink_sorting>0</filelink_sorting>
+    </tt_content>
+</dataset>

Tests/Functional/Fixtures/Searching/Facet.ts

@@ -0,0 +1,17 @@
+plugin {
+    tx_searchcore {
+        settings {
+            searching {
+                facets {
+                    contentTypes {
+                        terms {
+                            field = CType
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+module.tx_searchcore < plugin.tx_searchcore

Tests/Functional/FunctionalTests.xml

@@ -1,8 +1,7 @@
 <phpunit
     backupGlobals="true"
     backupStaticAttributes="false"
-    bootstrap="../../.Build/vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTestsBootstrap.php"
-
+    bootstrap="Bootstrap.php"
     colors="true"
     convertErrorsToExceptions="false"
     convertWarningsToExceptions="false"

Tests/Functional/Hooks/DataHandler/IgnoresUnkownOperationTest.php

@@ -1,54 +0,0 @@
-<?php
-namespace Codappix\SearchCore\Tests\Functional\Hooks\DataHandler;
-
-/*
- * Copyright (C) 2016  Daniel Siepmann <coding@daniel-siepmann.de>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301, USA.
- */
-
-use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
-use Codappix\SearchCore\Domain\Service\DataHandler as DataHandlerService;
-use Codappix\SearchCore\Hook\DataHandler as DataHandlerHook;
-use TYPO3\CMS\Core\DataHandling\DataHandler as Typo3DataHandler;
-use TYPO3\CMS\Core\Utility\GeneralUtility;
-use TYPO3\CMS\Extbase\Object\ObjectManager;
-
-class IgnoresUnkownOperationTest extends AbstractDataHandlerTest
-{
-    /**
-     * @var DataHandlerService|\PHPUnit_Framework_MockObject_MockObject|AccessibleObjectInterface
-     */
-    protected $subject;
-
-    /**
-     * @test
-     */
-    public function dataHandlerCommandSomethingIsIgnored()
-    {
-        $subject = new DataHandlerHook($this->subject);
-        $this->assertFalse(
-            $subject->processDatamap_afterDatabaseOperations(
-                'something',
-                'tt_content',
-                1,
-                [],
-                new Typo3DataHandler
-            ),
-            'Hook processed status "something".'
-        );
-    }
-}