Merge pull request #22 from DanielSiepmann/feature/add-rootline-blacklist

Feature/add rootline blacklist

Classes/Connection/Elasticsearch.php

@@ -94,7 +94,7 @@ class Elasticsearch implements Singleton, ConnectionInterface
     {
         $this->withType(
             $documentType,
-            function ($type) use($identifier) {
+            function ($type) use ($identifier) {
                 $type->deleteById($identifier);
             }
         );

Classes/Domain/Index/TcaIndexer.php

@@ -116,6 +116,7 @@ class TcaIndexer implements IndexerInterface
             '',
             (int) $offset . ',' . (int) $limit
         );
+        $this->tcaTableService->filterRecordsByRootLineBlacklist($records);
 
         foreach ($records as &$record) {
             $this->tcaTableService->prepareRecord($record);

Classes/Domain/Index/TcaIndexer/TcaTableService.php

@@ -23,6 +23,7 @@ namespace Leonmrni\SearchCore\Domain\Index\TcaIndexer;
 use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface;
 use Leonmrni\SearchCore\Domain\Index\IndexingException;
 use TYPO3\CMS\Backend\Utility\BackendUtility;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
 
 /**
  * Encapsulate logik related to TCA configuration.
@@ -104,6 +105,22 @@ class TcaTableService
         return $this->tableName . ' LEFT JOIN pages on ' . $this->tableName . '.pid = pages.uid';
     }
 
+    /**
+     * Filter the given records by root line blacklist settings.
+     *
+     * @param array &$records
+     * @return void
+     */
+    public function filterRecordsByRootLineBlacklist(array &$records)
+    {
+        $records = array_filter(
+            $records,
+            function ($record) {
+                return ! $this->isRecordBlacklistedByRootline($record);
+            }
+        );
+    }
+
     /**
      * Adjust record accordingly to configuration.
      * @param array &$record
@@ -125,7 +142,7 @@ class TcaTableService
      */
     public function getWhereClause()
     {
-        $whereClause = '1=1 '
+        $whereClause = '1=1'
             . BackendUtility::BEenableFields($this->tableName)
             . BackendUtility::deleteClause($this->tableName)
 
@@ -139,6 +156,15 @@ class TcaTableService
             $whereClause .= $userDefinedWhere;
         }
 
+        if ($this->isBlacklistedRootLineConfigured()) {
+            $whereClause .= ' AND pages.uid NOT IN ('
+                . implode(',', $this->getBlacklistedRootLine())
+                . ')'
+                . ' AND pages.pid NOT IN ('
+                . implode(',', $this->getBlacklistedRootLine())
+                . ')';
+        }
+
         $this->logger->debug('Generated where clause.', [$this->tableName, $whereClause]);
         return $whereClause;
     }
@@ -206,4 +232,47 @@ class TcaTableService
         return $this->tca['columns'][$columnName]['config'];
     }
 
+    /**
+     * Checks whether the given record was blacklisted by root line.
+     * This can be configured by typoscript as whole root lines can be black listed.
+     *
+     * @param array &$record
+     * @return bool
+     */
+    protected function isRecordBlacklistedByRootline(array &$record)
+    {
+        // NOTE: Does not support pages yet. We have to add a switch once we
+        // support them to use uid instead.
+        if (! $this->isBlackListedRootLineConfigured()) {
+            return false;
+        }
+
+        foreach (BackendUtility::BEgetRootLine($record['uid']) as $pageInRootLine) {
+            if (in_array($pageInRootLine['uid'], $this->getBlackListedRootLine())) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * Checks whether any page uids are black listed.
+     *
+     * @return bool
+     */
+    protected function isBlackListedRootLineConfigured()
+    {
+        return (bool) $this->configuration->getIfExists('index', 'rootLineBlacklist');
+    }
+
+    /**
+     * Get the list of black listed root line page uids.
+     *
+     * @return array<Int>
+     */
+    protected function getBlackListedRootLine()
+    {
+        return GeneralUtility::intExplode(',', $this->configuration->getIfExists('index', 'rootLineBlacklist'));
+    }
 }

Tests/Functional/AbstractFunctionalTestCase.php

@@ -24,18 +24,22 @@ use TYPO3\CMS\Core\Tests\FunctionalTestCase as CoreTestCase;
 
 /**
  * All functional tests should extend this base class.
- *
- * It will take care of leaving a clean environment for next test.
- * TODO: this is in reality an "elastica" abstract case - not search_core ;)
  */
 abstract class AbstractFunctionalTestCase extends CoreTestCase
 {
     protected $testExtensionsToLoad = ['typo3conf/ext/search_core'];
 
     /**
-     * @var \Elastica\Client
+     * Define whether to setup default typoscript on page 1.
+     *
+     * Set to false if you need to add further ts and use getDefaultPageTs to get the default one.
+     *
+     * This is necessary as setUpFrontendRootPage will allways add a new record
+     * and only the first one is used.
+     *
+     * @var bool
      */
-    protected $client;
+    protected $loadDefaultTs = true;
 
     public function setUp()
     {
@@ -46,19 +50,14 @@ abstract class AbstractFunctionalTestCase extends CoreTestCase
 
         // Provide necessary configuration for extension
         $this->importDataSet('Tests/Functional/Fixtures/BasicSetup.xml');
-        $this->setUpFrontendRootPage(1, ['EXT:search_core/Tests/Functional/Fixtures/BasicSetup.ts']);
 
-        // Create client to make requests and assert something.
+        if ($this->loadDefaultTs) {
-        $this->client = new \Elastica\Client([
+            $this->setUpFrontendRootPage(1, $this->getDefaultPageTs());
-            'host' => getenv('ES_HOST') ?: \Elastica\Connection::DEFAULT_HOST,
+        }
-            'port' => getenv('ES_PORT') ?: \Elastica\Connection::DEFAULT_PORT,
-        ]);
     }
 
-    public function tearDown()
+    protected function getDefaultPageTs()
     {
-        // Delete everything so next test starts clean.
+        return ['EXT:search_core/Tests/Functional/Fixtures/BasicSetup.ts'];
-        $this->client->getIndex('_all')->delete();
-        $this->client->getIndex('_all')->clearCache();
     }
 }

Tests/Functional/Connection/Elasticsearch/AbstractFunctionalTestCase.php

@@ -0,0 +1,54 @@
+<?php
+namespace Leonmrni\SearchCore\Tests\Functional\Connection\Elasticsearch;
+
+/*
+ * Copyright (C) 2016  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Leonmrni\SearchCore\Tests\Functional\AbstractFunctionalTestCase as BaseFunctionalTestCase;
+
+/**
+ * All functional tests should extend this base class.
+ *
+ * It will take care of leaving a clean environment for next test.
+ */
+abstract class AbstractFunctionalTestCase extends BaseFunctionalTestCase
+{
+    /**
+     * @var \Elastica\Client
+     */
+    protected $client;
+
+    public function setUp()
+    {
+        parent::setUp();
+
+        // Create client to make requests and assert something.
+        $this->client = new \Elastica\Client([
+            'host' => getenv('ES_HOST') ?: \Elastica\Connection::DEFAULT_HOST,
+            'port' => getenv('ES_PORT') ?: \Elastica\Connection::DEFAULT_PORT,
+        ]);
+    }
+
+    public function tearDown()
+    {
+        // Delete everything so next test starts clean.
+        $this->client->getIndex('_all')->delete();
+        $this->client->getIndex('_all')->clearCache();
+    }
+}

Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php

@@ -1,5 +1,5 @@
 <?php
-namespace Leonmrni\SearchCore\Tests\Functional\Indexing;
+namespace Leonmrni\SearchCore\Tests\Functional\Connection\Elasticsearch;
 
 /*
  * Copyright (C) 2016  Daniel Siepmann <coding@daniel-siepmann.de>
@@ -21,11 +21,10 @@ namespace Leonmrni\SearchCore\Tests\Functional\Indexing;
  */
 
 use Leonmrni\SearchCore\Domain\Index\IndexerFactory;
-use Leonmrni\SearchCore\Tests\Functional\AbstractFunctionalTestCase;
 use TYPO3\CMS\Extbase\Object\ObjectManager;
 
 /**
- *
+ * TODO: https://github.com/DanielSiepmann/search_core/issues/16
  */
 class IndexTcaTableTest extends AbstractFunctionalTestCase
 {

Tests/Functional/Fixtures/Indexing/TcaIndexer/RespectRootLineBlacklist.ts

@@ -0,0 +1,11 @@
+plugin {
+    tx_searchcore {
+        settings {
+            index {
+                rootLineBlacklist = 3
+            }
+        }
+    }
+}
+
+module.tx_searchcore < plugin.tx_searchcore

Tests/Functional/Fixtures/Indexing/TcaIndexer/RespectRootLineBlacklist.xml

@@ -0,0 +1,100 @@
+<?xml version="1.0" encoding="utf-8"?>
+<dataset>
+    <pages>
+        <uid>2</uid>
+        <pid>1</pid>
+        <title>Content here will be indexed</title>
+    </pages>
+
+    <pages>
+        <uid>3</uid>
+        <pid>2</pid>
+        <title>Content here will not be indexed</title>
+    </pages>
+
+    <pages>
+        <uid>4</uid>
+        <pid>3</pid>
+        <title>Content here will not be indexed either</title>
+    </pages>
+
+    <tt_content>
+        <uid>1</uid>
+        <pid>1</pid>
+        <tstamp>1480686370</tstamp>
+        <crdate>1480686370</crdate>
+        <hidden>0</hidden>
+        <sorting>72</sorting>
+        <CType>textmedia</CType>
+        <header>indexed content element</header>
+        <bodytext>this is the content of textmedia content element that should get indexed</bodytext>
+        <media>0</media>
+        <layout>0</layout>
+        <deleted>0</deleted>
+        <cols>0</cols>
+        <starttime>0</starttime>
+        <endtime>0</endtime>
+        <colPos>0</colPos>
+        <filelink_sorting>0</filelink_sorting>
+    </tt_content>
+
+    <tt_content>
+        <uid>2</uid>
+        <pid>2</pid>
+        <tstamp>1480686371</tstamp>
+        <crdate>1480686370</crdate>
+        <hidden>0</hidden>
+        <sorting>72</sorting>
+        <CType>textmedia</CType>
+        <header>indexed content element</header>
+        <bodytext>this is the content of textmedia content element that should get indexed</bodytext>
+        <media>0</media>
+        <layout>0</layout>
+        <deleted>0</deleted>
+        <cols>0</cols>
+        <starttime>0</starttime>
+        <endtime>0</endtime>
+        <colPos>0</colPos>
+        <filelink_sorting>0</filelink_sorting>
+    </tt_content>
+
+    <tt_content>
+        <uid>3</uid>
+        <pid>3</pid>
+        <tstamp>1480686371</tstamp>
+        <crdate>1480686370</crdate>
+        <hidden>0</hidden>
+        <sorting>72</sorting>
+        <CType>textmedia</CType>
+        <header>This content should not be indexed due to root line contraints</header>
+        <bodytext></bodytext>
+        <media>0</media>
+        <layout>0</layout>
+        <deleted>0</deleted>
+        <cols>0</cols>
+        <starttime>0</starttime>
+        <endtime>0</endtime>
+        <colPos>0</colPos>
+        <filelink_sorting>0</filelink_sorting>
+    </tt_content>
+
+    <tt_content>
+        <uid>4</uid>
+        <pid>4</pid>
+        <tstamp>1480686371</tstamp>
+        <crdate>1480686370</crdate>
+        <hidden>0</hidden>
+        <sorting>72</sorting>
+        <CType>textmedia</CType>
+        <header>This content should not be indexed due to root line contraints</header>
+        <bodytext></bodytext>
+        <media>0</media>
+        <layout>0</layout>
+        <deleted>0</deleted>
+        <cols>0</cols>
+        <starttime>0</starttime>
+        <endtime>0</endtime>
+        <colPos>0</colPos>
+        <filelink_sorting>0</filelink_sorting>
+    </tt_content>
+</dataset>

Tests/Functional/Hooks/DataHandlerTest.php

@@ -21,12 +21,15 @@ namespace Leonmrni\SearchCore\Tests\Functional\Hooks;
  */
 
 use Leonmrni\SearchCore\Hook\DataHandler as Hook;
-use Leonmrni\SearchCore\Tests\Functional\AbstractFunctionalTestCase;
+use Leonmrni\SearchCore\Tests\Functional\Connection\Elasticsearch\AbstractFunctionalTestCase;
 use TYPO3\CMS\Core\DataHandling\DataHandler as CoreDataHandler;
 use TYPO3\CMS\Extbase\Object\ObjectManager;
 
 /**
- *
+ * TODO: Rewrite as this test doesn't test what it should do.
+ * We have to split it up in two tests:
+ * 1. Test whether TYPO3 DataHandler will our hook as expected.
+ * 2. Test whether our hook will send the documents to connection as expected.
  */
 class DataHandlerTest extends AbstractFunctionalTestCase
 {

Tests/Functional/Indexing/TcaIndexerTest.php

@@ -0,0 +1,86 @@
+<?php
+namespace Leonmrni\SearchCore\Tests\Indexing;
+
+/*
+ * Copyright (C) 2016  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface;
+use Leonmrni\SearchCore\Connection\Elasticsearch;
+use Leonmrni\SearchCore\Domain\Index\TcaIndexer;
+use Leonmrni\SearchCore\Domain\Index\TcaIndexer\RelationResolver;
+use Leonmrni\SearchCore\Domain\Index\TcaIndexer\TcaTableService;
+use Leonmrni\SearchCore\Tests\Functional\AbstractFunctionalTestCase;
+use TYPO3\CMS\Extbase\Object\ObjectManager;
+
+class TcaIndexerTest extends AbstractFunctionalTestCase
+{
+    protected $loadDefaultTs = false;
+
+    /**
+     * @test
+     */
+    public function respectRootLineBlacklist()
+    {
+        $this->importDataSet('Tests/Functional/Fixtures/Indexing/TcaIndexer/RespectRootLineBlacklist.xml');
+        $this->setUpFrontendRootPage(
+            1,
+            array_merge(
+                $this->getDefaultPageTs(),
+                ['EXT:search_core/Tests/Functional/Fixtures/Indexing/TcaIndexer/RespectRootLineBlacklist.ts']
+            )
+        );
+
+        $objectManager = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class);
+        $tableName = 'tt_content';
+        $tableService = $objectManager->get(
+            TcaTableService::class,
+            $tableName,
+            $objectManager->get(RelationResolver::class),
+            $objectManager->get(ConfigurationContainerInterface::class)
+        );
+        $connection = $this->getAccessibleMock(
+            Elasticsearch::class,
+            [
+                'addDocuments',
+            ],
+            [],
+            '',
+            false
+        );
+
+        $connection->expects($this->once())
+            ->method('addDocuments')
+            ->with(
+                $this->stringContains('tt_content'),
+                $this->callback(function ($documents) {
+                    foreach ($documents as $document) {
+                        // Page uids 1 and 2 are allowed while 3 and 4 are not allowed.
+                        // Therefore only documents with page uid 1 and 2 should exist.
+                        if (! isset($document['pid']) || ! in_array($document['pid'], [1, 2])) {
+                            return false;
+                        }
+                    }
+
+                    return true;
+                })
+            );
+
+        $objectManager->get(TcaIndexer::class, $tableService, $connection)->indexAllDocuments();
+    }
+}