Merge remote-tracking branch 'origin/develop' into feature/cms-8-support

Classes/Connection/Elasticsearch.php

@@ -189,7 +189,7 @@ class Elasticsearch implements Singleton, ConnectionInterface
         $search->addIndex('typo3content');
         $search->setQuery($this->queryFactory->create($searchRequest));
 
-        return $this->objectManager->get(SearchResult::class, $search->search());
+        return $this->objectManager->get(SearchResult::class, $searchRequest, $search->search());
     }
 
     /**

Classes/Connection/Elasticsearch/SearchResult.php

@@ -22,11 +22,17 @@ namespace Codappix\SearchCore\Connection\Elasticsearch;
 
 use Codappix\SearchCore\Connection\FacetInterface;
 use Codappix\SearchCore\Connection\ResultItemInterface;
+use Codappix\SearchCore\Connection\SearchRequestInterface;
 use Codappix\SearchCore\Connection\SearchResultInterface;
 use TYPO3\CMS\Extbase\Object\ObjectManagerInterface;
 
 class SearchResult implements SearchResultInterface
 {
+    /**
+     * @var SearchRequestInterface
+     */
+    protected $searchRequest;
+
     /**
      * @var \Elastica\ResultSet
      */
@@ -47,8 +53,12 @@ class SearchResult implements SearchResultInterface
      */
     protected $objectManager;
 
-    public function __construct(\Elastica\ResultSet $result, ObjectManagerInterface $objectManager)
+    public function __construct(
-    {
+        SearchRequestInterface $searchRequest,
+        \Elastica\ResultSet $result,
+        ObjectManagerInterface $objectManager
+    ) {
+        $this->searchRequest = $searchRequest;
         $this->result = $result;
         $this->objectManager = $objectManager;
     }
@@ -75,25 +85,37 @@ class SearchResult implements SearchResultInterface
         return $this->facets;
     }
 
-    /**
+    public function getCurrentCount()
-     * Returns the total sum of matching results.
-     *
-     * @return int
-     */
-    public function getTotalCount()
     {
-        return $this->result->getTotalHits();
+        return $this->result->count();
+    }
+
+    protected function initResults()
+    {
+        if ($this->results !== []) {
+            return;
+        }
+
+        foreach ($this->result->getResults() as $result) {
+            $this->results[] = new ResultItem($result);
+        }
+    }
+
+    protected function initFacets()
+    {
+        if ($this->facets !== [] || !$this->result->hasAggregations()) {
+            return;
+        }
+
+        foreach ($this->result->getAggregations() as $aggregationName => $aggregation) {
+            $this->facets[$aggregationName] = $this->objectManager->get(Facet::class, $aggregationName, $aggregation);
+        }
     }
 
     // Countable - Interface
-    /**
-     * Returns the total sum of results contained in this result.
-     *
-     * @return int
-     */
     public function count()
     {
-        return $this->result->count();
+        return $this->result->getTotalHits();
     }
 
     // Iterator - Interface
@@ -122,25 +144,41 @@ class SearchResult implements SearchResultInterface
         $this->result->rewind();
     }
 
-    protected function initResults()
+    // Extbase QueryResultInterface - Implemented to support Pagination of Fluid.
-    {
-        if ($this->results !== []) {
-            return;
-        }
 
-        foreach ($this->result->getResults() as $result) {
+    public function getQuery()
-            $this->results[] = new ResultItem($result);
+    {
-        }
+        return $this->searchRequest;
     }
 
-    protected function initFacets()
+    public function getFirst()
     {
-        if ($this->facets !== [] || !$this->result->hasAggregations()) {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502195121);
-            return;
+    }
-        }
 
-        foreach ($this->result->getAggregations() as $aggregationName => $aggregation) {
+    public function toArray()
-            $this->facets[$aggregationName] = $this->objectManager->get(Facet::class, $aggregationName, $aggregation);
+    {
-        }
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502195135);
+    }
+
+    public function offsetExists($offset)
+    {
+        // Return false to allow Fluid to use appropriate getter methods.
+        return false;
+    }
+
+    public function offsetGet($offset)
+    {
+        throw new \BadMethodCallException('Use getter to fetch properties.', 1502196933);
+    }
+
+    public function offsetSet($offset, $value)
+    {
+        throw new \BadMethodCallException('You are not allowed to modify the result.', 1502196934);
+    }
+
+    public function offsetUnset($offset)
+    {
+        throw new \BadMethodCallException('You are not allowed to modify the result.', 1502196936);
     }
 }

Classes/Connection/SearchRequestInterface.php

@@ -20,10 +20,9 @@ namespace Codappix\SearchCore\Connection;
  * 02110-1301, USA.
  */
 
-/**
+use TYPO3\CMS\Extbase\Persistence\QueryInterface;
- *
+
- */
+interface SearchRequestInterface extends QueryInterface
-interface SearchRequestInterface
 {
     /**
      * Returns the actual string the user searched for.
@@ -41,11 +40,4 @@ interface SearchRequestInterface
      * @return array
      */
     public function getFilter();
-
-    /**
-     * Defines how many results should be fetched.
-     *
-     * @return int
-     */
-    public function getSize();
 }

Classes/Connection/SearchResultInterface.php

@@ -20,10 +20,12 @@ namespace Codappix\SearchCore\Connection;
  * 02110-1301, USA.
  */
 
+use TYPO3\CMS\Extbase\Persistence\QueryResultInterface;
+
 /**
  * A search result.
  */
-interface SearchResultInterface extends \Iterator, \Countable
+interface SearchResultInterface extends \Iterator, \Countable, QueryResultInterface
 {
     /**
      * @return array<ResultItemInterface>
@@ -38,18 +40,9 @@ interface SearchResultInterface extends \Iterator, \Countable
     public function getFacets();
 
     /**
-     * Returns the total sum of matching results.
+     * Returns the number of results in current result
      *
      * @return int
      */
-    public function getTotalCount();
+    public function getCurrentCount();
-
-    // Countable - Interface
-
-    /**
-     * Returns the total sum of results contained in this result.
-     *
-     * @return int
-     */
-    public function count();
 }

Classes/Domain/Index/TcaIndexer/TcaTableService.php

@@ -26,6 +26,8 @@ use Codappix\SearchCore\Database\Doctrine\Where;
 use Codappix\SearchCore\Domain\Index\IndexingException;
 use TYPO3\CMS\Backend\Utility\BackendUtility;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Core\Utility\RootlineUtility;
+use TYPO3\CMS\Extbase\Object\ObjectManagerInterface;
 
 /**
  * Encapsulate logik related to TCA configuration.
@@ -49,15 +51,20 @@ class TcaTableService
      */
     protected $configuration;
 
+    /**
+     * @var RelationResolver
+     */
+    protected $relationResolver;
+
     /**
      * @var \TYPO3\CMS\Core\Log\Logger
      */
     protected $logger;
 
     /**
-     * @var RelationResolver
+     * @var ObjectManagerInterface
      */
-    protected $relationResolver;
+    protected $objectManager;
 
     /**
      * Inject log manager to get concrete logger from it.
@@ -69,6 +76,14 @@ class TcaTableService
         $this->logger = $logManager->getLogger(__CLASS__);
     }
 
+    /**
+     * @param ObjectManagerInterface $objectManager
+     */
+    public function injectObjectManager(ObjectManagerInterface $objectManager)
+    {
+        $this->objectManager = $objectManager;
+    }
+
     /**
      * @param string $tableName
      * @param ConfigurationContainerInterface $configuration
@@ -255,26 +270,59 @@ class TcaTableService
      * Checks whether the given record was blacklisted by root line.
      * This can be configured by typoscript as whole root lines can be black listed.
      *
-     * NOTE: Does not support pages yet. We have to add a switch once we
+     * Also further TYPO3 mechanics are taken into account. Does a valid root
-     * support them to use uid instead.
+     * line exist, is page inside a recycler, is inherited start- endtime
+     * excluded, etc.
      *
      * @param array &$record
      * @return bool
      */
     protected function isRecordBlacklistedByRootline(array &$record) : bool
     {
-        // If no rootline exists, the record is on a unreachable page and therefore blacklisted.
+        $pageUid = $record['pid'];
-        $rootline = BackendUtility::BEgetRootLine($record['pid']);
+        if ($this->tableName === 'pages') {
-        if (!isset($rootline[0])) {
+            $pageUid = $record['uid'];
+        }
+
+        try {
+            $rootline = $this->objectManager->get(RootlineUtility::class, $pageUid)->get();
+        } catch (\RuntimeException $e) {
+            $this->logger->notice(
+                sprintf('Could not fetch rootline for page %u, because: %s', $pageUid, $e->getMessage()),
+                [$record, $e]
+            );
             return true;
         }
 
-        // Check configured black list if present.
+        foreach ($rootline as $pageInRootLine) {
-        if ($this->isBlackListedRootLineConfigured()) {
+            // Check configured black list if present.
-            foreach ($rootline as $pageInRootLine) {
+            if ($this->isBlackListedRootLineConfigured()
-                if (in_array($pageInRootLine['uid'], $this->getBlackListedRootLine())) {
+                && in_array($pageInRootLine['uid'], $this->getBlackListedRootLine())
-                    return true;
+            ) {
-                }
+                $this->logger->info(
+                    sprintf(
+                        'Record %u is black listed due to configured root line configuration of page %u.',
+                        $record['uid'],
+                        $pageInRootLine['uid']
+                    ),
+                    [$record, $pageInRootLine]
+                );
+                return true;
+            }
+
+            if ($pageInRootLine['extendToSubpages'] && (
+                ($pageInRootLine['endtime'] > 0 && $pageInRootLine['endtime'] <= time())
+                || ($pageInRootLine['starttime'] > 0 && $pageInRootLine['starttime'] >= time())
+            )) {
+                $this->logger->info(
+                    sprintf(
+                        'Record %u is black listed due to configured timing of parent page %u.',
+                        $record['uid'],
+                        $pageInRootLine['uid']
+                    ),
+                    [$record, $pageInRootLine]
+                );
+                return true;
             }
         }
 

Classes/Domain/Model/SearchRequest.php

@@ -20,6 +20,7 @@ namespace Codappix\SearchCore\Domain\Model;
  * 02110-1301, USA.
  */
 
+use Codappix\SearchCore\Connection\ConnectionInterface;
 use Codappix\SearchCore\Connection\FacetRequestInterface;
 use Codappix\SearchCore\Connection\SearchRequestInterface;
 
@@ -35,11 +36,6 @@ class SearchRequest implements SearchRequestInterface
      */
     protected $query = '';
 
-    /**
-     * @var int
-     */
-    protected $size = 10;
-
     /**
      * @var array
      */
@@ -50,6 +46,23 @@ class SearchRequest implements SearchRequestInterface
      */
     protected $facets = [];
 
+    /**
+     * @var int
+     */
+    protected $offset = 0;
+
+    /**
+     * @var int
+     */
+    protected $limit = 10;
+
+    /**
+     * Used for QueryInterface implementation to allow execute method to work.
+     *
+     * @var ConnectionInterface
+     */
+    protected $connection = null;
+
     /**
      * @param string $query
      */
@@ -119,18 +132,162 @@ class SearchRequest implements SearchRequestInterface
     }
 
     /**
-     * @return int
+     * Define connection to use for this request.
+     * Necessary to allow implementation of execute for interface.
+     *
+     * @param ConnectionInterface $connection
      */
-    public function getSize()
+    public function setConnection(ConnectionInterface $connection)
     {
-        return $this->size;
+        $this->connection = $connection;
     }
 
-    /**
+    // Extbase QueryInterface
-     * @param int $size
+    // Current implementation covers only paginate widget support.
-     */
+    public function execute($returnRawQueryResult = false)
-    public function setSize($size)
     {
-        $this->size = (int) $size;
+        if ($this->connection instanceof ConnectionInterface) {
+            return $this->connection->search($this);
+        }
+
+        throw new \InvalidArgumentException(
+            'Connection was not set before, therefore execute can not work. Use `setConnection` before.',
+            1502197732
+        );
+    }
+
+    public function setLimit($limit)
+    {
+        $this->limit = (int) $limit;
+    }
+
+    public function setOffset($offset)
+    {
+        $this->offset = (int) $offset;
+    }
+
+    public function getLimit()
+    {
+        return $this->limit;
+    }
+
+    public function getOffset()
+    {
+        return $this->offset;
+    }
+
+    public function getSource()
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196146);
+    }
+
+    public function setOrderings(array $orderings)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196163);
+    }
+
+    public function matching($constraint)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196197);
+    }
+
+    public function logicalAnd($constraint1)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196166);
+    }
+
+    public function logicalOr($constraint1)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196198);
+    }
+
+    public function logicalNot(\TYPO3\CMS\Extbase\Persistence\Generic\Qom\ConstraintInterface $constraint)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196166);
+    }
+
+    public function equals($propertyName, $operand, $caseSensitive = true)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196199);
+    }
+
+    public function like($propertyName, $operand, $caseSensitive = true)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196167);
+    }
+
+    public function contains($propertyName, $operand)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196200);
+    }
+
+    public function in($propertyName, $operand)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196167);
+    }
+
+    public function lessThan($propertyName, $operand)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196201);
+    }
+
+    public function lessThanOrEqual($propertyName, $operand)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196168);
+    }
+
+    public function greaterThan($propertyName, $operand)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196202);
+    }
+
+    public function greaterThanOrEqual($propertyName, $operand)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196168);
+    }
+
+    public function getType()
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196203);
+    }
+
+    public function setQuerySettings(\TYPO3\CMS\Extbase\Persistence\Generic\QuerySettingsInterface $querySettings)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196168);
+    }
+
+    public function getQuerySettings()
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196205);
+    }
+
+    public function count()
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196169);
+    }
+
+    public function getOrderings()
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196206);
+    }
+
+    public function getConstraint()
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196171);
+    }
+
+    public function isEmpty($propertyName)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196207);
+    }
+
+    public function setSource(\TYPO3\CMS\Extbase\Persistence\Generic\Qom\SourceInterface $source)
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196172);
+    }
+
+    public function getStatement()
+    {
+        throw new \BadMethodCallException('Method is not implemented yet.', 1502196208);
     }
 }

Classes/Domain/Search/QueryFactory.php

@@ -94,8 +94,8 @@ class QueryFactory
     protected function addSize(SearchRequestInterface $searchRequest, array &$query)
     {
         $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [
-            'from' => 0,
+            'from' => $searchRequest->getOffset(),
-            'size' => $searchRequest->getSize(),
+            'size' => $searchRequest->getLimit(),
         ]);
     }
 

Classes/Domain/Search/SearchService.php

@@ -69,6 +69,7 @@ class SearchService
      */
     public function search(SearchRequestInterface $searchRequest)
     {
+        $searchRequest->setConnection($this->connection);
         $this->addSize($searchRequest);
         $this->addConfiguredFacets($searchRequest);
 
@@ -82,7 +83,7 @@ class SearchService
      */
     protected function addSize(SearchRequestInterface $searchRequest)
     {
-        $searchRequest->setSize(
+        $searchRequest->setLimit(
             $this->configuration->getIfExists('searching.size') ?: 10
         );
     }

Tests/Functional/Fixtures/Indexing/PagesIndexer/BrokenRootLine.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="utf-8"?>
+<dataset>
+    <!-- DISABLED PAGES -->
+    <pages>
+        <uid>3</uid>
+        <pid>2</pid>
+        <title>Some disabled page due broken root line</title>
+    </pages>
+    <pages>
+        <uid>4</uid>
+        <pid>3</pid>
+        <title>Some disabled page due to parent pages root line being broken</title>
+    </pages>
+    <!-- ENABLED PAGES -->
+    <pages>
+        <uid>6</uid>
+        <pid>1</pid>
+        <title>Some enabled page due valid root line</title>
+    </pages>
+</dataset>

Tests/Functional/Fixtures/Indexing/PagesIndexer/InheritedTiming.xml

@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="utf-8"?>
+<dataset>
+    <!-- DISABLED PAGES -->
+    <pages>
+        <uid>2</uid>
+        <pid>1</pid>
+        <title>Some disabled page due to timing</title>
+        <endtime>1502186635</endtime>
+        <extendToSubpages>1</extendToSubpages>
+    </pages>
+    <pages>
+        <uid>3</uid>
+        <pid>2</pid>
+        <title>Some disabled page due to inherited timing</title>
+    </pages>
+    <pages>
+        <uid>4</uid>
+        <pid>1</pid>
+        <title>Some disabled page due to timing</title>
+        <starttime>2147483647</starttime>
+        <extendToSubpages>1</extendToSubpages>
+    </pages>
+    <pages>
+        <uid>5</uid>
+        <pid>4</pid>
+        <title>Some disabled page due to inherited timing</title>
+    </pages>
+    <!-- ENABLED PAGES -->
+    <pages>
+        <uid>6</uid>
+        <pid>1</pid>
+        <title>Some enabled page due to no be below inherited disabled timing</title>
+    </pages>
+</dataset>

Tests/Functional/Fixtures/Indexing/PagesIndexer/Recycler.xml

@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="utf-8"?>
+<dataset>
+    <!-- DISABLED PAGES -->
+    <pages>
+        <uid>2</uid>
+        <pid>1</pid>
+        <title>Some disabled page due being recycler</title>
+        <doktype>255</doktype>
+    </pages>
+    <pages>
+        <uid>3</uid>
+        <pid>2</pid>
+        <title>Some disabled page due to parent page being recycler</title>
+    </pages>
+    <!-- ENABLED PAGES -->
+    <pages>
+        <uid>6</uid>
+        <pid>1</pid>
+        <title>Some enabled page due to no be below recycler</title>
+    </pages>
+</dataset>

Tests/Functional/Indexing/PagesIndexerTest.php

@@ -61,4 +61,44 @@ class PagesIndexerTest extends AbstractFunctionalTestCase
         $this->inject($indexer, 'connection', $connection);
         $indexer->indexAllDocuments();
     }
+
+    /**
+     * @test
+     * @dataProvider rootLineDataSets
+     * @param string $dataSetPath
+     */
+    public function rootLineIsRespectedDuringIndexing($dataSetPath)
+    {
+        $this->importDataSet($dataSetPath);
+
+        $objectManager = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class);
+        $tableName = 'pages';
+
+        $connection = $this->getMockBuilder(Elasticsearch::class)
+            ->setMethods(['addDocuments'])
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $connection->expects($this->once())
+            ->method('addDocuments')
+            ->with(
+                $this->stringContains($tableName),
+                $this->callback(function ($documents) {
+                    return count($documents) === 2;
+                })
+            );
+
+        $indexer = $objectManager->get(IndexerFactory::class)->getIndexer($tableName);
+        $this->inject($indexer, 'connection', $connection);
+        $indexer->indexAllDocuments();
+    }
+
+    public function rootLineDataSets()
+    {
+        return [
+            'Broken root line' => ['Tests/Functional/Fixtures/Indexing/PagesIndexer/BrokenRootLine.xml'],
+            'Recycler doktype' => ['Tests/Functional/Fixtures/Indexing/PagesIndexer/Recycler.xml'],
+            'Extended timing to sub pages' => ['Tests/Functional/Fixtures/Indexing/PagesIndexer/InheritedTiming.xml'],
+        ];
+    }
 }

Tests/Unit/Domain/Search/QueryFactoryTest.php

@@ -157,13 +157,19 @@ class QueryFactoryTest extends AbstractUnitTestCase
             ->method('get')
             ->will($this->throwException(new InvalidArgumentException));
         $searchRequest = new SearchRequest('SearchWord');
-        $searchRequest->setSize(45);
+        $searchRequest->setLimit(45);
+        $searchRequest->setOffset(35);
 
         $query = $this->subject->create($searchRequest);
         $this->assertSame(
             45,
             $query->toArray()['size'],
-            'Size was not added to query.'
+            'Limit was not added to query.'
+        );
+        $this->assertSame(
+            35,
+            $query->toArray()['from'],
+            'From was not added to query.'
         );
     }
 

Tests/Unit/Domain/Search/SearchServiceTest.php

@@ -67,7 +67,7 @@ class SearchServiceTest extends AbstractUnitTestCase
         $this->connection->expects($this->once())
             ->method('search')
             ->with($this->callback(function ($searchRequest) {
-                return $searchRequest->getSize() === 45;
+                return $searchRequest->getLimit() === 45;
             }));
 
         $searchRequest = new SearchRequest('SearchWord');
@@ -86,7 +86,7 @@ class SearchServiceTest extends AbstractUnitTestCase
         $this->connection->expects($this->once())
             ->method('search')
             ->with($this->callback(function ($searchRequest) {
-                return $searchRequest->getSize() === 10;
+                return $searchRequest->getLimit() === 10;
             }));
 
         $searchRequest = new SearchRequest('SearchWord');