From 7178c400b18feff12e75a37ec0073d85e8dc17b2 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 6 Jul 2017 23:53:56 +0200 Subject: [PATCH 01/75] TASK: Rename vendor As we move it from private repo to codappix. --- Classes/Command/IndexCommandController.php | 6 +++--- Classes/Configuration/ConfigurationContainer.php | 2 +- .../ConfigurationContainerInterface.php | 2 +- Classes/Configuration/InvalidArgumentException.php | 2 +- Classes/Configuration/NoConfigurationException.php | 2 +- Classes/Connection/ConnectionInterface.php | 2 +- Classes/Connection/Elasticsearch.php | 2 +- Classes/Connection/Elasticsearch/Connection.php | 4 ++-- .../Connection/Elasticsearch/DocumentFactory.php | 2 +- Classes/Connection/Elasticsearch/IndexFactory.php | 2 +- Classes/Connection/Elasticsearch/SearchResult.php | 4 ++-- Classes/Connection/Elasticsearch/TypeFactory.php | 2 +- Classes/Connection/SearchRequestInterface.php | 2 +- Classes/Connection/SearchResultInterface.php | 2 +- Classes/Controller/SearchController.php | 6 +++--- Classes/Domain/Index/IndexerFactory.php | 2 +- Classes/Domain/Index/IndexerInterface.php | 2 +- Classes/Domain/Index/IndexingException.php | 2 +- Classes/Domain/Index/NoRecordFoundException.php | 2 +- Classes/Domain/Index/TcaIndexer.php | 4 ++-- .../Index/TcaIndexer/InvalidArgumentException.php | 2 +- .../Domain/Index/TcaIndexer/RelationResolver.php | 2 +- .../Domain/Index/TcaIndexer/TcaTableService.php | 6 +++--- Classes/Domain/Model/SearchRequest.php | 4 ++-- Classes/Domain/Search/SearchService.php | 10 +++++----- Classes/Domain/Service/DataHandler.php | 8 ++++---- Classes/Hook/DataHandler.php | 6 +++--- Documentation/source/conf.py | 8 ++++---- Documentation/source/readme.rst | 11 +++++------ Tests/Functional/AbstractFunctionalTestCase.php | 2 +- .../Elasticsearch/AbstractFunctionalTestCase.php | 4 ++-- .../Connection/Elasticsearch/IndexTcaTableTest.php | 6 +++--- .../Hooks/DataHandler/AbstractDataHandlerTest.php | 10 +++++----- .../DataHandler/IgnoresUnkownOperationTest.php | 8 ++++---- .../Hooks/DataHandler/NonAllowedTablesTest.php | 8 ++++---- ...lowedTablesWithMultipleTablesConfiguredTest.php | 2 +- .../DataHandler/ProcessesAllowedTablesTest.php | 8 ++++---- ...lowedTablesWithMultipleTablesConfiguredTest.php | 2 +- .../Indexing/TcaIndexer/RelationResolverTest.php | 6 +++--- Tests/Functional/Indexing/TcaIndexerTest.php | 14 +++++++------- composer.json | 10 +++++----- ext_emconf.php | 2 +- ext_localconf.php | 12 ++++++------ ext_tables.php | 2 +- 44 files changed, 103 insertions(+), 104 deletions(-) diff --git a/Classes/Command/IndexCommandController.php b/Classes/Command/IndexCommandController.php index c1ade27..2a89a88 100644 --- a/Classes/Command/IndexCommandController.php +++ b/Classes/Command/IndexCommandController.php @@ -1,5 +1,5 @@ @@ -20,7 +20,7 @@ namespace Leonmrni\SearchCore\Command; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Domain\Index\IndexerFactory; +use Codappix\SearchCore\Domain\Index\IndexerFactory; use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Extbase\Mvc\Controller\CommandController; @@ -35,7 +35,7 @@ class IndexCommandController extends CommandController protected $indexerFactory; /** - * @var \Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface + * @var \Codappix\SearchCore\Configuration\ConfigurationContainerInterface * @inject */ protected $configuration; diff --git a/Classes/Configuration/ConfigurationContainer.php b/Classes/Configuration/ConfigurationContainer.php index db66a10..7481692 100644 --- a/Classes/Configuration/ConfigurationContainer.php +++ b/Classes/Configuration/ConfigurationContainer.php @@ -1,5 +1,5 @@ diff --git a/Classes/Configuration/ConfigurationContainerInterface.php b/Classes/Configuration/ConfigurationContainerInterface.php index 7bc45ef..87e273e 100644 --- a/Classes/Configuration/ConfigurationContainerInterface.php +++ b/Classes/Configuration/ConfigurationContainerInterface.php @@ -1,5 +1,5 @@ diff --git a/Classes/Configuration/InvalidArgumentException.php b/Classes/Configuration/InvalidArgumentException.php index fd9e048..1f5f8f0 100644 --- a/Classes/Configuration/InvalidArgumentException.php +++ b/Classes/Configuration/InvalidArgumentException.php @@ -1,5 +1,5 @@ diff --git a/Classes/Configuration/NoConfigurationException.php b/Classes/Configuration/NoConfigurationException.php index bc6c1ba..db4e511 100644 --- a/Classes/Configuration/NoConfigurationException.php +++ b/Classes/Configuration/NoConfigurationException.php @@ -1,5 +1,5 @@ diff --git a/Classes/Connection/ConnectionInterface.php b/Classes/Connection/ConnectionInterface.php index fdd09e3..a130f9e 100644 --- a/Classes/Connection/ConnectionInterface.php +++ b/Classes/Connection/ConnectionInterface.php @@ -1,5 +1,5 @@ diff --git a/Classes/Connection/Elasticsearch.php b/Classes/Connection/Elasticsearch.php index 560ca07..c40a5e2 100644 --- a/Classes/Connection/Elasticsearch.php +++ b/Classes/Connection/Elasticsearch.php @@ -1,5 +1,5 @@ diff --git a/Classes/Connection/Elasticsearch/Connection.php b/Classes/Connection/Elasticsearch/Connection.php index 25c7361..cd0a1fd 100644 --- a/Classes/Connection/Elasticsearch/Connection.php +++ b/Classes/Connection/Elasticsearch/Connection.php @@ -1,5 +1,5 @@ @@ -20,7 +20,7 @@ namespace Leonmrni\SearchCore\Connection\Elasticsearch; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use TYPO3\CMS\Core\SingletonInterface as Singleton; /** diff --git a/Classes/Connection/Elasticsearch/DocumentFactory.php b/Classes/Connection/Elasticsearch/DocumentFactory.php index 43462fe..99d29e5 100644 --- a/Classes/Connection/Elasticsearch/DocumentFactory.php +++ b/Classes/Connection/Elasticsearch/DocumentFactory.php @@ -1,5 +1,5 @@ diff --git a/Classes/Connection/Elasticsearch/IndexFactory.php b/Classes/Connection/Elasticsearch/IndexFactory.php index 0a06ad4..019b091 100644 --- a/Classes/Connection/Elasticsearch/IndexFactory.php +++ b/Classes/Connection/Elasticsearch/IndexFactory.php @@ -1,5 +1,5 @@ diff --git a/Classes/Connection/Elasticsearch/SearchResult.php b/Classes/Connection/Elasticsearch/SearchResult.php index 3f90d60..48c26cb 100644 --- a/Classes/Connection/Elasticsearch/SearchResult.php +++ b/Classes/Connection/Elasticsearch/SearchResult.php @@ -1,5 +1,5 @@ @@ -20,7 +20,7 @@ namespace Leonmrni\SearchCore\Connection\Elasticsearch; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Connection\SearchResultInterface; +use Codappix\SearchCore\Connection\SearchResultInterface; /** * diff --git a/Classes/Connection/Elasticsearch/TypeFactory.php b/Classes/Connection/Elasticsearch/TypeFactory.php index b529937..d5283f8 100644 --- a/Classes/Connection/Elasticsearch/TypeFactory.php +++ b/Classes/Connection/Elasticsearch/TypeFactory.php @@ -1,5 +1,5 @@ diff --git a/Classes/Connection/SearchRequestInterface.php b/Classes/Connection/SearchRequestInterface.php index ecf7a74..fd98c1f 100644 --- a/Classes/Connection/SearchRequestInterface.php +++ b/Classes/Connection/SearchRequestInterface.php @@ -1,5 +1,5 @@ diff --git a/Classes/Connection/SearchResultInterface.php b/Classes/Connection/SearchResultInterface.php index f00e97c..2dfdfc6 100644 --- a/Classes/Connection/SearchResultInterface.php +++ b/Classes/Connection/SearchResultInterface.php @@ -1,5 +1,5 @@ diff --git a/Classes/Controller/SearchController.php b/Classes/Controller/SearchController.php index b7624f7..068f9a1 100644 --- a/Classes/Controller/SearchController.php +++ b/Classes/Controller/SearchController.php @@ -1,5 +1,5 @@ @@ -20,8 +20,8 @@ namespace Leonmrni\SearchCore\Controller; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Domain\Model\SearchRequest; -use Leonmrni\SearchCore\Domain\Search\SearchService; +use Codappix\SearchCore\Domain\Model\SearchRequest; +use Codappix\SearchCore\Domain\Search\SearchService; use TYPO3\CMS\Extbase\Mvc\Controller\ActionController; /** diff --git a/Classes/Domain/Index/IndexerFactory.php b/Classes/Domain/Index/IndexerFactory.php index 4ecbfb9..3d3f460 100644 --- a/Classes/Domain/Index/IndexerFactory.php +++ b/Classes/Domain/Index/IndexerFactory.php @@ -1,5 +1,5 @@ diff --git a/Classes/Domain/Index/IndexerInterface.php b/Classes/Domain/Index/IndexerInterface.php index d70b410..5fef64f 100644 --- a/Classes/Domain/Index/IndexerInterface.php +++ b/Classes/Domain/Index/IndexerInterface.php @@ -1,5 +1,5 @@ diff --git a/Classes/Domain/Index/IndexingException.php b/Classes/Domain/Index/IndexingException.php index a22660d..d8ca71e 100644 --- a/Classes/Domain/Index/IndexingException.php +++ b/Classes/Domain/Index/IndexingException.php @@ -1,5 +1,5 @@ diff --git a/Classes/Domain/Index/NoRecordFoundException.php b/Classes/Domain/Index/NoRecordFoundException.php index 8a92408..469359d 100644 --- a/Classes/Domain/Index/NoRecordFoundException.php +++ b/Classes/Domain/Index/NoRecordFoundException.php @@ -1,5 +1,5 @@ diff --git a/Classes/Domain/Index/TcaIndexer.php b/Classes/Domain/Index/TcaIndexer.php index b50c6fa..9d192d3 100644 --- a/Classes/Domain/Index/TcaIndexer.php +++ b/Classes/Domain/Index/TcaIndexer.php @@ -1,5 +1,5 @@ @@ -21,7 +21,7 @@ namespace Leonmrni\SearchCore\Domain\Index; */ use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; -use Leonmrni\SearchCore\Connection\ConnectionInterface; +use Codappix\SearchCore\Connection\ConnectionInterface; /** * Will index the given table using configuration from TCA. diff --git a/Classes/Domain/Index/TcaIndexer/InvalidArgumentException.php b/Classes/Domain/Index/TcaIndexer/InvalidArgumentException.php index 0f3dc9a..bc2036f 100644 --- a/Classes/Domain/Index/TcaIndexer/InvalidArgumentException.php +++ b/Classes/Domain/Index/TcaIndexer/InvalidArgumentException.php @@ -1,5 +1,5 @@ diff --git a/Classes/Domain/Index/TcaIndexer/RelationResolver.php b/Classes/Domain/Index/TcaIndexer/RelationResolver.php index 69eb4a2..b09e483 100644 --- a/Classes/Domain/Index/TcaIndexer/RelationResolver.php +++ b/Classes/Domain/Index/TcaIndexer/RelationResolver.php @@ -1,5 +1,5 @@ diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index 936d425..b7b73bc 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -1,5 +1,5 @@ @@ -20,8 +20,8 @@ namespace Leonmrni\SearchCore\Domain\Index\TcaIndexer; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; -use Leonmrni\SearchCore\Domain\Index\IndexingException; +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Domain\Index\IndexingException; use TYPO3\CMS\Backend\Utility\BackendUtility; use TYPO3\CMS\Core\Utility\GeneralUtility; diff --git a/Classes/Domain/Model/SearchRequest.php b/Classes/Domain/Model/SearchRequest.php index 1de2f71..0f8a98b 100644 --- a/Classes/Domain/Model/SearchRequest.php +++ b/Classes/Domain/Model/SearchRequest.php @@ -1,5 +1,5 @@ @@ -20,7 +20,7 @@ namespace Leonmrni\SearchCore\Domain\Model; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Connection\SearchRequestInterface; +use Codappix\SearchCore\Connection\SearchRequestInterface; /** * Represents a search request used to process an actual search. diff --git a/Classes/Domain/Search/SearchService.php b/Classes/Domain/Search/SearchService.php index afbaf7e..3f6375f 100644 --- a/Classes/Domain/Search/SearchService.php +++ b/Classes/Domain/Search/SearchService.php @@ -1,5 +1,5 @@ @@ -20,10 +20,10 @@ namespace Leonmrni\SearchCore\Domain\Search; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Connection\ConnectionInterface; -use Leonmrni\SearchCore\Connection\SearchRequestInterface; -use Leonmrni\SearchCore\Connection\SearchResultInterface; -use Leonmrni\SearchCore\Domain\Model\SearchRequest; +use Codappix\SearchCore\Connection\ConnectionInterface; +use Codappix\SearchCore\Connection\SearchRequestInterface; +use Codappix\SearchCore\Connection\SearchResultInterface; +use Codappix\SearchCore\Domain\Model\SearchRequest; /** * Service to process a search request. diff --git a/Classes/Domain/Service/DataHandler.php b/Classes/Domain/Service/DataHandler.php index 213b9cc..06b287c 100644 --- a/Classes/Domain/Service/DataHandler.php +++ b/Classes/Domain/Service/DataHandler.php @@ -1,5 +1,5 @@ @@ -20,7 +20,7 @@ namespace Leonmrni\SearchCore\Domain\Service; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use TYPO3\CMS\Core\SingletonInterface as Singleton; use TYPO3\CMS\Core\Utility\GeneralUtility; @@ -40,13 +40,13 @@ class DataHandler implements Singleton /** * TODO: Only inject on first use?! * - * @var \Leonmrni\SearchCore\Connection\ConnectionInterface + * @var \Codappix\SearchCore\Connection\ConnectionInterface * @inject */ protected $connection; /** - * @var \Leonmrni\SearchCore\Domain\Index\IndexerFactory + * @var \Codappix\SearchCore\Domain\Index\IndexerFactory * @inject */ protected $indexerFactory; diff --git a/Classes/Hook/DataHandler.php b/Classes/Hook/DataHandler.php index 1bcb4a5..54e20ab 100644 --- a/Classes/Hook/DataHandler.php +++ b/Classes/Hook/DataHandler.php @@ -1,5 +1,5 @@ @@ -20,8 +20,8 @@ namespace Leonmrni\SearchCore\Hook; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Configuration\NoConfigurationException; -use Leonmrni\SearchCore\Domain\Service\DataHandler as OwnDataHandler; +use Codappix\SearchCore\Configuration\NoConfigurationException; +use Codappix\SearchCore\Domain\Service\DataHandler as OwnDataHandler; use TYPO3\CMS\Backend\Utility\BackendUtility; use TYPO3\CMS\Core\DataHandling\DataHandler as CoreDataHandler; use TYPO3\CMS\Core\Log\LogManager; diff --git a/Documentation/source/conf.py b/Documentation/source/conf.py index 07129e7..a708209 100644 --- a/Documentation/source/conf.py +++ b/Documentation/source/conf.py @@ -120,7 +120,7 @@ html_theme = 'alabaster' # documentation. html_theme_options = { 'description': 'TYPO3 Extension to integrate search services.', - 'github_user': 'DanielSiepmann', + 'github_user': 'Codappix', 'github_repo': 'search_core', 'github_button': True, 'github_banner': True, @@ -306,7 +306,7 @@ intersphinx_mapping = { 't3tcaref': ('https://docs.typo3.org/typo3cms/TCAReference/', None), } extlinks = { - 'project': ('https://github.com/DanielSiepmann/search_core/projects/%s', 'Github project: '), - 'pr': ('https://github.com/DanielSiepmann/search_core/pull/%s', 'Github pull request: '), - 'issue': ('https://github.com/DanielSiepmann/search_core/issues/%s', 'Github issue: '), + 'project': ('https://github.com/Codappix/search_core/projects/%s', 'Github project: '), + 'pr': ('https://github.com/Codappix/search_core/pull/%s', 'Github pull request: '), + 'issue': ('https://github.com/Codappix/search_core/issues/%s', 'Github issue: '), } diff --git a/Documentation/source/readme.rst b/Documentation/source/readme.rst index ec5e0e0..16448a9 100644 --- a/Documentation/source/readme.rst +++ b/Documentation/source/readme.rst @@ -23,9 +23,8 @@ This is still a very early alpha version. More information can be taken from Git We are also focusing on Code Quality and Testing through `travis ci`_, `scrutinizer`_ and `codacy`_. -.. _current issues: https://github.com/DanielSiepmann/search_core/issues -.. _current projects: https://github.com/DanielSiepmann/search_core/projects -.. _travis ci: https://travis-ci.org/DanielSiepmann/search_core -.. _scrutinizer: https://scrutinizer-ci.com/g/DanielSiepmann/search_core/inspections -.. _codacy: https://www.codacy.com/app/daniel-siepmann/search_core/dashboard - +.. _current issues: https://github.com/Codappix/search_core/issues +.. _current projects: https://github.com/Codappix/search_core/projects +.. _travis ci: https://travis-ci.org/Codappix/search_core +.. _scrutinizer: https://scrutinizer-ci.com/g/Codappix/search_core/inspections +.. _codacy: https://www.codacy.com/app/Codappix/search_core/dashboard diff --git a/Tests/Functional/AbstractFunctionalTestCase.php b/Tests/Functional/AbstractFunctionalTestCase.php index c231784..7f808c7 100644 --- a/Tests/Functional/AbstractFunctionalTestCase.php +++ b/Tests/Functional/AbstractFunctionalTestCase.php @@ -1,5 +1,5 @@ diff --git a/Tests/Functional/Connection/Elasticsearch/AbstractFunctionalTestCase.php b/Tests/Functional/Connection/Elasticsearch/AbstractFunctionalTestCase.php index cdd8ddf..5097fc3 100644 --- a/Tests/Functional/Connection/Elasticsearch/AbstractFunctionalTestCase.php +++ b/Tests/Functional/Connection/Elasticsearch/AbstractFunctionalTestCase.php @@ -1,5 +1,5 @@ @@ -20,7 +20,7 @@ namespace Leonmrni\SearchCore\Tests\Functional\Connection\Elasticsearch; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Tests\Functional\AbstractFunctionalTestCase as BaseFunctionalTestCase; +use Codappix\SearchCore\Tests\Functional\AbstractFunctionalTestCase as BaseFunctionalTestCase; /** * All functional tests should extend this base class. diff --git a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php index 5e0c1a7..9678d83 100644 --- a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php +++ b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php @@ -1,5 +1,5 @@ @@ -20,7 +20,7 @@ namespace Leonmrni\SearchCore\Tests\Functional\Connection\Elasticsearch; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Domain\Index\IndexerFactory; +use Codappix\SearchCore\Domain\Index\IndexerFactory; use TYPO3\CMS\Extbase\Object\ObjectManager; /** @@ -61,7 +61,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase /** * @test - * @expectedException \Leonmrni\SearchCore\Domain\Index\IndexingException + * @expectedException \Codappix\SearchCore\Domain\Index\IndexingException */ public function indexingNonConfiguredTableWillThrowException() { diff --git a/Tests/Functional/Hooks/DataHandler/AbstractDataHandlerTest.php b/Tests/Functional/Hooks/DataHandler/AbstractDataHandlerTest.php index a26021d..087c5bc 100644 --- a/Tests/Functional/Hooks/DataHandler/AbstractDataHandlerTest.php +++ b/Tests/Functional/Hooks/DataHandler/AbstractDataHandlerTest.php @@ -1,5 +1,5 @@ @@ -20,10 +20,10 @@ namespace Leonmrni\SearchCore\Tests\Functional\Hooks\DataHandler; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; -use Leonmrni\SearchCore\Domain\Service\DataHandler as DataHandlerService; -use Leonmrni\SearchCore\Hook\DataHandler as DataHandlerHook; -use Leonmrni\SearchCore\Tests\Functional\AbstractFunctionalTestCase; +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Domain\Service\DataHandler as DataHandlerService; +use Codappix\SearchCore\Hook\DataHandler as DataHandlerHook; +use Codappix\SearchCore\Tests\Functional\AbstractFunctionalTestCase; use TYPO3\CMS\Core\DataHandling\DataHandler as Typo3DataHandler; use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Extbase\Object\ObjectManager; diff --git a/Tests/Functional/Hooks/DataHandler/IgnoresUnkownOperationTest.php b/Tests/Functional/Hooks/DataHandler/IgnoresUnkownOperationTest.php index 44f5186..b1676e3 100644 --- a/Tests/Functional/Hooks/DataHandler/IgnoresUnkownOperationTest.php +++ b/Tests/Functional/Hooks/DataHandler/IgnoresUnkownOperationTest.php @@ -1,5 +1,5 @@ @@ -20,9 +20,9 @@ namespace Leonmrni\SearchCore\Tests\Functional\Hooks\DataHandler; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; -use Leonmrni\SearchCore\Domain\Service\DataHandler as DataHandlerService; -use Leonmrni\SearchCore\Hook\DataHandler as DataHandlerHook; +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Domain\Service\DataHandler as DataHandlerService; +use Codappix\SearchCore\Hook\DataHandler as DataHandlerHook; use TYPO3\CMS\Core\DataHandling\DataHandler as Typo3DataHandler; use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Extbase\Object\ObjectManager; diff --git a/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php b/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php index 6073633..c33701d 100644 --- a/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php +++ b/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php @@ -1,5 +1,5 @@ @@ -20,9 +20,9 @@ namespace Leonmrni\SearchCore\Tests\Functional\Hooks\DataHandler; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; -use Leonmrni\SearchCore\Domain\Service\DataHandler as DataHandlerService; -use Leonmrni\SearchCore\Hook\DataHandler as DataHandlerHook; +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Domain\Service\DataHandler as DataHandlerService; +use Codappix\SearchCore\Hook\DataHandler as DataHandlerHook; use TYPO3\CMS\Core\DataHandling\DataHandler as Typo3DataHandler; use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Extbase\Object\ObjectManager; diff --git a/Tests/Functional/Hooks/DataHandler/NonAllowedTablesWithMultipleTablesConfiguredTest.php b/Tests/Functional/Hooks/DataHandler/NonAllowedTablesWithMultipleTablesConfiguredTest.php index 0bdf563..ebbee99 100644 --- a/Tests/Functional/Hooks/DataHandler/NonAllowedTablesWithMultipleTablesConfiguredTest.php +++ b/Tests/Functional/Hooks/DataHandler/NonAllowedTablesWithMultipleTablesConfiguredTest.php @@ -1,5 +1,5 @@ diff --git a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php index e9e834e..715fe29 100644 --- a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php +++ b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php @@ -1,5 +1,5 @@ @@ -20,9 +20,9 @@ namespace Leonmrni\SearchCore\Tests\Functional\Hooks\DataHandler; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; -use Leonmrni\SearchCore\Domain\Service\DataHandler as DataHandlerService; -use Leonmrni\SearchCore\Hook\DataHandler as DataHandlerHook; +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Domain\Service\DataHandler as DataHandlerService; +use Codappix\SearchCore\Hook\DataHandler as DataHandlerHook; use TYPO3\CMS\Core\DataHandling\DataHandler as Typo3DataHandler; use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Extbase\Object\ObjectManager; diff --git a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesWithMultipleTablesConfiguredTest.php b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesWithMultipleTablesConfiguredTest.php index f743bdb..c0b0aa3 100644 --- a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesWithMultipleTablesConfiguredTest.php +++ b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesWithMultipleTablesConfiguredTest.php @@ -1,5 +1,5 @@ diff --git a/Tests/Functional/Indexing/TcaIndexer/RelationResolverTest.php b/Tests/Functional/Indexing/TcaIndexer/RelationResolverTest.php index 360cacd..5577ac0 100644 --- a/Tests/Functional/Indexing/TcaIndexer/RelationResolverTest.php +++ b/Tests/Functional/Indexing/TcaIndexer/RelationResolverTest.php @@ -1,5 +1,5 @@ @@ -20,8 +20,8 @@ namespace Leonmrni\SearchCore\Tests\Indexing\TcaIndexer; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Domain\Index\TcaIndexer\TcaTableService; -use Leonmrni\SearchCore\Tests\Functional\AbstractFunctionalTestCase; +use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService; +use Codappix\SearchCore\Tests\Functional\AbstractFunctionalTestCase; use TYPO3\CMS\Backend\Utility\BackendUtility; use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Extbase\Object\ObjectManager; diff --git a/Tests/Functional/Indexing/TcaIndexerTest.php b/Tests/Functional/Indexing/TcaIndexerTest.php index 056af39..2b3f817 100644 --- a/Tests/Functional/Indexing/TcaIndexerTest.php +++ b/Tests/Functional/Indexing/TcaIndexerTest.php @@ -1,5 +1,5 @@ @@ -20,12 +20,12 @@ namespace Leonmrni\SearchCore\Tests\Indexing; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; -use Leonmrni\SearchCore\Connection\Elasticsearch; -use Leonmrni\SearchCore\Domain\Index\TcaIndexer; -use Leonmrni\SearchCore\Domain\Index\TcaIndexer\RelationResolver; -use Leonmrni\SearchCore\Domain\Index\TcaIndexer\TcaTableService; -use Leonmrni\SearchCore\Tests\Functional\AbstractFunctionalTestCase; +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Connection\Elasticsearch; +use Codappix\SearchCore\Domain\Index\TcaIndexer; +use Codappix\SearchCore\Domain\Index\TcaIndexer\RelationResolver; +use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService; +use Codappix\SearchCore\Tests\Functional\AbstractFunctionalTestCase; use TYPO3\CMS\Extbase\Object\ObjectManager; class TcaIndexerTest extends AbstractFunctionalTestCase diff --git a/composer.json b/composer.json index debb71f..61aedce 100644 --- a/composer.json +++ b/composer.json @@ -1,17 +1,17 @@ { - "name": "leonmrni/search_core", + "name": "codappix/search_core", "type": "typo3-cms-extension", - "description": "Leonmrni Search Core.", - "homepage": "http://www.leonmrni.com", + "description": "Codappix Search Core.", + "homepage": "https://github.com/Codappix/search_core", "license": ["GPL-2.0+"], "autoload": { "psr-4": { - "Leonmrni\\SearchCore\\": "Classes" + "Codappix\\SearchCore\\": "Classes" } }, "autoload-dev": { "psr-4": { - "Leonmrni\\SearchCore\\Tests\\": "Tests/", + "Codappix\\SearchCore\\Tests\\": "Tests/", "TYPO3\\CMS\\Core\\Tests\\": ".Build/vendor/typo3/cms/typo3/sysext/core/Tests/" } }, diff --git a/ext_emconf.php b/ext_emconf.php index 3aab0c7..de4fd8c 100644 --- a/ext_emconf.php +++ b/ext_emconf.php @@ -13,7 +13,7 @@ $EM_CONF[$_EXTKEY] = [ ], 'autoload' => [ 'psr-4' => [ - 'Leonmrni\\SearchCore\\' => 'Classes', + 'Codappix\\SearchCore\\' => 'Classes', ], ], 'state' => 'alpha', diff --git a/ext_localconf.php b/ext_localconf.php index 7495d73..1c1f9cd 100644 --- a/ext_localconf.php +++ b/ext_localconf.php @@ -11,15 +11,15 @@ call_user_func( 'SC_OPTIONS' => [ 'extbase' => [ 'commandControllers' => [ - Leonmrni\SearchCore\Command\IndexCommandController::class, + Codappix\SearchCore\Command\IndexCommandController::class, ], ], 't3lib/class.t3lib_tcemain.php' => [ 'processCmdmapClass' => [ - $extensionKey => '&' . \Leonmrni\SearchCore\Hook\DataHandler::class, + $extensionKey => '&' . \Codappix\SearchCore\Hook\DataHandler::class, ], 'processDatamapClass' => [ - $extensionKey => '&' . \Leonmrni\SearchCore\Hook\DataHandler::class, + $extensionKey => '&' . \Codappix\SearchCore\Hook\DataHandler::class, ], ], ], @@ -27,7 +27,7 @@ call_user_func( ); TYPO3\CMS\Extbase\Utility\ExtensionUtility::configurePlugin( - 'Leonmrni.' . $extensionKey, + 'Codappix.' . $extensionKey, 'search', [ 'Search' => 'search' @@ -39,8 +39,8 @@ call_user_func( \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('TYPO3\CMS\Extbase\Object\Container\Container') ->registerImplementation( - 'Leonmrni\SearchCore\Connection\ConnectionInterface', - 'Leonmrni\SearchCore\Connection\Elasticsearch' + 'Codappix\SearchCore\Connection\ConnectionInterface', + 'Codappix\SearchCore\Connection\Elasticsearch' ); }, $_EXTKEY diff --git a/ext_tables.php b/ext_tables.php index d6d07ed..1a54787 100644 --- a/ext_tables.php +++ b/ext_tables.php @@ -7,7 +7,7 @@ ); TYPO3\CMS\Extbase\Utility\ExtensionUtility::registerPlugin( - 'Leonmrni.' . $_EXTKEY, + 'Codappix.' . $_EXTKEY, 'search', 'Search Core' ); From b7b783a7fed644ac6c302cc0c8103f719b2721c3 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sat, 11 Nov 2017 16:46:03 +0100 Subject: [PATCH 02/75] TASK: Use Code Sniffer at travis This way we need no external service. Each developer can fully run all tests and cgl on local environment. Also this integrated better into IDEs and editors. --- .travis.yml | 1 + Classes/Connection/Elasticsearch.php | 5 +++- .../Index/TcaIndexer/TcaTableService.php | 9 ++++++-- Classes/Domain/Search/QueryFactory.php | 11 +++++++-- Classes/Hook/DataHandler.php | 9 ++++++-- Makefile | 5 +++- .../ConfigurationUtilityTest.php | 7 ++++-- Tests/Unit/Domain/Search/QueryFactoryTest.php | 1 - composer.json | 11 ++++----- phpcs.xml.dist | 23 +++++++++++++++++++ 10 files changed, 64 insertions(+), 18 deletions(-) create mode 100644 phpcs.xml.dist diff --git a/.travis.yml b/.travis.yml index 4490198..0adb894 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,6 +33,7 @@ services: install: make install script: + - make cgl - make unitTests - make functionalTests diff --git a/Classes/Connection/Elasticsearch.php b/Classes/Connection/Elasticsearch.php index 8a3cb2b..b86bebf 100644 --- a/Classes/Connection/Elasticsearch.php +++ b/Classes/Connection/Elasticsearch.php @@ -132,7 +132,10 @@ class Elasticsearch implements Singleton, ConnectionInterface } ); } catch (\Elastica\Exception\NotFoundException $exception) { - $this->logger->debug('Tried to delete document in index, which does not exist.', [$documentType, $identifier]); + $this->logger->debug( + 'Tried to delete document in index, which does not exist.', + [$documentType, $identifier] + ); } } diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index 7a322a0..bf631b1 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -159,7 +159,9 @@ class TcaTableService $parameters = []; $whereClause = $this->getSystemWhereClause(); - $userDefinedWhere = $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.additionalWhereClause'); + $userDefinedWhere = $this->configuration->getIfExists( + 'indexing.' . $this->getTableName() . '.additionalWhereClause' + ); if (is_string($userDefinedWhere)) { $whereClause .= ' AND ' . $userDefinedWhere; } @@ -361,6 +363,9 @@ class TcaTableService */ protected function getBlackListedRootLine() : array { - return GeneralUtility::intExplode(',', $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist')); + return GeneralUtility::intExplode( + ',', + $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist') + ); } } diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index f73372d..5879b9f 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -163,7 +163,11 @@ class QueryFactory { try { $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ - 'stored_fields' => GeneralUtility::trimExplode(',', $this->configuration->get('searching.fields.stored_fields'), true), + 'stored_fields' => GeneralUtility::trimExplode( + ',', + $this->configuration->get('searching.fields.stored_fields'), + true + ), ]); } catch (InvalidArgumentException $e) { // Nothing configured @@ -171,7 +175,10 @@ class QueryFactory try { $scriptFields = $this->configuration->get('searching.fields.script_fields'); - $scriptFields = $this->configurationUtility->replaceArrayValuesWithRequestContent($searchRequest, $scriptFields); + $scriptFields = $this->configurationUtility->replaceArrayValuesWithRequestContent( + $searchRequest, + $scriptFields + ); $scriptFields = $this->configurationUtility->filterByCondition($scriptFields); if ($scriptFields !== []) { $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['script_fields' => $scriptFields]); diff --git a/Classes/Hook/DataHandler.php b/Classes/Hook/DataHandler.php index d0eb1ba..0caac7c 100644 --- a/Classes/Hook/DataHandler.php +++ b/Classes/Hook/DataHandler.php @@ -103,8 +103,13 @@ class DataHandler implements Singleton * * @return bool False if hook was not processed. */ - public function processDatamap_afterDatabaseOperations($status, $table, $uid, array $fieldArray, CoreDataHandler $dataHandler) - { + public function processDatamap_afterDatabaseOperations( + $status, + $table, + $uid, + array $fieldArray, + CoreDataHandler $dataHandler + ) { if (! $this->shouldProcessHookForTable($table)) { $this->logger->debug('Database update not processed.', [$table, $uid]); return false; diff --git a/Makefile b/Makefile index b6bc72b..b8f4ac5 100644 --- a/Makefile +++ b/Makefile @@ -12,9 +12,12 @@ typo3DatabaseHost ?= "127.0.0.1" .PHONY: install install: clean - COMPOSER_PROCESS_TIMEOUT=1000 composer require -vv --dev --prefer-dist --ignore-platform-reqs typo3/cms="$(TYPO3_VERSION)" + COMPOSER_PROCESS_TIMEOUT=1000 composer require -vv --dev --prefer-dist typo3/cms="$(TYPO3_VERSION)" git checkout composer.json +cgl: + ./.Build/bin/phpcs + functionalTests: typo3DatabaseName=$(typo3DatabaseName) \ typo3DatabaseUsername=$(typo3DatabaseUsername) \ diff --git a/Tests/Unit/Configuration/ConfigurationUtilityTest.php b/Tests/Unit/Configuration/ConfigurationUtilityTest.php index b2dc5f8..4db367c 100644 --- a/Tests/Unit/Configuration/ConfigurationUtilityTest.php +++ b/Tests/Unit/Configuration/ConfigurationUtilityTest.php @@ -31,8 +31,11 @@ class ConfigurationUtilityTest extends AbstractUnitTestCase * @test * @dataProvider possibleRequestAndConfigurationForFluidtemplate */ - public function recursiveEntriesAreProcessedAsFluidtemplate(SearchRequestInterface $searchRequest, array $array, array $expected) - { + public function recursiveEntriesAreProcessedAsFluidtemplate( + SearchRequestInterface $searchRequest, + array $array, + array $expected + ) { $subject = new ConfigurationUtility(); $this->assertSame( diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index 6301cdd..218f8e5 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -465,7 +465,6 @@ class QueryFactoryTest extends AbstractUnitTestCase $query->toArray()['script_fields'], 'Script fields were not added to query as expected.' ); - } /** diff --git a/composer.json b/composer.json index 985be01..efa5146 100644 --- a/composer.json +++ b/composer.json @@ -16,13 +16,13 @@ } }, "require" : { - "php": ">=7.1.0", - "typo3/cms": "~8.7", + "php": ">=5.6.0", "ruflin/elastica": "~3.2" }, "require-dev": { - "typo3/testing-framework": "~1.1.0", - "phpunit/phpunit": "~6.2.0" + "phpunit/phpunit": "~5.7.0", + "squizlabs/php_codesniffer": "~3.1.1", + "typo3/cms": "~7.6" }, "config": { "optimize-autoloader": true, @@ -36,9 +36,6 @@ ] }, "extra": { - "branch-alias": { - "dev-develop": "1.0.x-dev" - }, "typo3/cms": { "cms-package-dir": "{$vendor-dir}/typo3/cms", "web-dir": ".Build/web" diff --git a/phpcs.xml.dist b/phpcs.xml.dist new file mode 100644 index 0000000..e722968 --- /dev/null +++ b/phpcs.xml.dist @@ -0,0 +1,23 @@ + + + The coding standard for search_core. + + Classes/ + Tests/ + + + + + + + + + + + + + + + Classes/Hook/DataHandler.php + + From 0bac2df6a4bb8636e15db523486818c03343dabb Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sat, 11 Nov 2017 17:01:30 +0100 Subject: [PATCH 03/75] BUGFIX: Fix broken dependencies --- composer.json | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/composer.json b/composer.json index efa5146..2761473 100644 --- a/composer.json +++ b/composer.json @@ -16,13 +16,14 @@ } }, "require" : { - "php": ">=5.6.0", + "php": ">=7.1.0", + "typo3/cms": "~8.7", "ruflin/elastica": "~3.2" }, "require-dev": { - "phpunit/phpunit": "~5.7.0", - "squizlabs/php_codesniffer": "~3.1.1", - "typo3/cms": "~7.6" + "phpunit/phpunit": "~6.4.4", + "typo3/testing-framework": "~1.1.5", + "squizlabs/php_codesniffer": "~3.1.1" }, "config": { "optimize-autoloader": true, From 33d4116fe6a024e05fadc7cad8eae5600c11a847 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Mon, 29 Jan 2018 22:45:53 +0100 Subject: [PATCH 04/75] TASK: Fix license for packagist --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 2761473..f3bd4c3 100644 --- a/composer.json +++ b/composer.json @@ -3,7 +3,7 @@ "type": "typo3-cms-extension", "description": "Codappix Search Core.", "homepage": "https://github.com/Codappix/search_core", - "license": ["GPL-2.0+"], + "license": ["GPL-2.0-or-later"], "autoload": { "psr-4": { "Codappix\\SearchCore\\": "Classes" From ebf722a03ec0b969133ac1c61a0d4fed8796c6ea Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Mon, 29 Jan 2018 22:45:53 +0100 Subject: [PATCH 05/75] TASK: Fix license for packagist --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 61aedce..9cfe8b3 100644 --- a/composer.json +++ b/composer.json @@ -3,7 +3,7 @@ "type": "typo3-cms-extension", "description": "Codappix Search Core.", "homepage": "https://github.com/Codappix/search_core", - "license": ["GPL-2.0+"], + "license": ["GPL-2.0-or-later"], "autoload": { "psr-4": { "Codappix\\SearchCore\\": "Classes" From 92af364b8dbde0f9e633ef1e4ebcec72639c6360 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 22 Feb 2018 20:49:25 +0100 Subject: [PATCH 06/75] FEATURE: Allow to disable elasticsearch integration This extension currently ships with Elasticsearch integration which is enabled by default. This behaviour is kept for backwards compatibility. Still you now have the possibility to disable this integration in extension manager, just check the "disable" box for elasticsearch. In the future elasticsearch will become another extension and no default is shipped with search_core. But for now, as we are still in alpha / beta phase we keep things together to keep development fast. Resolves: #111 --- Documentation/source/configuration.rst | 5 ++++- Documentation/source/installation.rst | 4 ++++ ext_conf_template.txt | 4 ++++ ext_localconf.php | 17 ++++++++++++----- 4 files changed, 24 insertions(+), 6 deletions(-) create mode 100644 ext_conf_template.txt diff --git a/Documentation/source/configuration.rst b/Documentation/source/configuration.rst index eca5ba4..09289ac 100644 --- a/Documentation/source/configuration.rst +++ b/Documentation/source/configuration.rst @@ -5,9 +5,12 @@ Configuration ============= +Installation wide configuration is handled inside of the extension manager. Just check out the +options there, they all have labels. + The extension offers the following configuration options through TypoScript. If you overwrite them through `setup` make sure to keep them in the `module` area as they will be accessed from backend -mode of TYPO3. Do so by placing the following line at the end:: +mode of TYPO3 for indexing. Do so by placing the following line at the end:: module.tx_searchcore < plugin.tx_searchcore diff --git a/Documentation/source/installation.rst b/Documentation/source/installation.rst index ddfb065..5bb0c56 100644 --- a/Documentation/source/installation.rst +++ b/Documentation/source/installation.rst @@ -19,4 +19,8 @@ In that case you need to install all dependencies yourself. Dependencies are: Afterwards you need to enable the extension through the extension manager and include the static TypoScript setup. +If you **don't** want to use the included elasticsearch integration, you have to disable it in the +extension manager configuration of the extension by checking the checkbox. +It's currently enabled by default but will be moved into its own extension in the future. + .. _downloading: https://github.com/DanielSiepmann/search_core/archive/master.zip diff --git a/ext_conf_template.txt b/ext_conf_template.txt new file mode 100644 index 0000000..5ddadb4 --- /dev/null +++ b/ext_conf_template.txt @@ -0,0 +1,4 @@ +disable { + # cat=basic/enable; type=boolean; label=Disable Elasticsearch, which is enabled by default + elasticsearch = true +} diff --git a/ext_localconf.php b/ext_localconf.php index c52d05d..a713d0b 100644 --- a/ext_localconf.php +++ b/ext_localconf.php @@ -37,11 +37,18 @@ call_user_func( ] ); - \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('TYPO3\CMS\Extbase\Object\Container\Container') - ->registerImplementation( - 'Codappix\SearchCore\Connection\ConnectionInterface', - 'Codappix\SearchCore\Connection\Elasticsearch' - ); + // API does make use of object manager, therefore use GLOBALS + $extensionConfiguration = unserialize($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf'][$extensionKey]); + if ($extensionConfiguration === false + || !isset($extensionConfiguration['disable.']['elasticsearch']) + || $extensionConfiguration['disable.']['elasticsearch'] !== '1' + ) { + \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Extbase\Object\Container\Container::class) + ->registerImplementation( + \Codappix\SearchCore\Connection\ConnectionInterface::class, + \Codappix\SearchCore\Connection\Elasticsearch::class + ); + } }, $_EXTKEY ); From ebaeaf4c92e224618211723d80f3acd6ba73fb68 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 22 Feb 2018 21:59:13 +0100 Subject: [PATCH 07/75] TASK: Support PHP 7.0 As some (e.g. debian) do not provide PHP 7.1 and we did not use so much features which were introduced in PHP 7.1, we add support for PHP 7.0. --- .travis.yml | 1 + Classes/Domain/Index/TcaIndexer/RelationResolver.php | 2 +- Classes/Domain/Index/TcaIndexer/TcaTableService.php | 4 ++-- Makefile | 2 +- composer.json | 4 ++-- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4490198..7414a0e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,7 @@ before_install: language: php php: + - 7.0 - 7.1 env: diff --git a/Classes/Domain/Index/TcaIndexer/RelationResolver.php b/Classes/Domain/Index/TcaIndexer/RelationResolver.php index a2302ae..fb19a4f 100644 --- a/Classes/Domain/Index/TcaIndexer/RelationResolver.php +++ b/Classes/Domain/Index/TcaIndexer/RelationResolver.php @@ -33,7 +33,7 @@ use TYPO3\CMS\Core\Utility\GeneralUtility; */ class RelationResolver implements Singleton { - public function resolveRelationsForRecord(TcaTableService $service, array &$record) : void + public function resolveRelationsForRecord(TcaTableService $service, array &$record) { foreach (array_keys($record) as $column) { // TODO: Define / configure fields to exclude?! diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index 7a322a0..f059248 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -129,7 +129,7 @@ class TcaTableService * @param array &$records * @return void */ - public function filterRecordsByRootLineBlacklist(array &$records) : void + public function filterRecordsByRootLineBlacklist(array &$records) { $records = array_filter( $records, @@ -142,7 +142,7 @@ class TcaTableService /** * @param array &$record */ - public function prepareRecord(array &$record) : void + public function prepareRecord(array &$record) { $this->relationResolver->resolveRelationsForRecord($this, $record); diff --git a/Makefile b/Makefile index b6bc72b..4fb0ce4 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ typo3DatabaseHost ?= "127.0.0.1" .PHONY: install install: clean - COMPOSER_PROCESS_TIMEOUT=1000 composer require -vv --dev --prefer-dist --ignore-platform-reqs typo3/cms="$(TYPO3_VERSION)" + COMPOSER_PROCESS_TIMEOUT=1000 composer require -vv --dev --prefer-dist typo3/cms="$(TYPO3_VERSION)" git checkout composer.json functionalTests: diff --git a/composer.json b/composer.json index b704e77..c260226 100644 --- a/composer.json +++ b/composer.json @@ -15,8 +15,8 @@ "TYPO3\\CMS\\Core\\Tests\\": ".Build/vendor/typo3/cms/typo3/sysext/core/Tests/" } }, - "require" : { - "php": ">=7.1.0", + "require": { + "php": ">=7.0.0", "typo3/cms": "~8.7", "ruflin/elastica": "~3.2" }, From 47b32820346282fe7969e904ebf18583d0b8c1ae Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 22 Feb 2018 21:55:52 +0100 Subject: [PATCH 08/75] BUGFIX: Allow indexing of new records with their relations Relations were inserted by TYPO3's DataHandler after indexing and were therefore not indexed. We now use a later hook after DataHandler has finished everything, so we know that we can index. As it's not relevant, we do not differentiate between add and update anymore, as both trigger "indexDocument" internal. Resolves: #112 --- Classes/Domain/Service/DataHandler.php | 10 ---- Classes/Hook/DataHandler.php | 50 ++++++++--------- .../Form/Finisher/DataHandlerFinisher.php | 4 +- .../IgnoresUnkownOperationTest.php | 54 ------------------- .../DataHandler/NonAllowedTablesTest.php | 6 +-- .../ProcessesAllowedTablesTest.php | 6 +-- .../Form/Finisher/DataHandlerFinisherTest.php | 11 ++-- 7 files changed, 32 insertions(+), 109 deletions(-) delete mode 100644 Tests/Functional/Hooks/DataHandler/IgnoresUnkownOperationTest.php diff --git a/Classes/Domain/Service/DataHandler.php b/Classes/Domain/Service/DataHandler.php index 6ac8069..10a01fe 100644 --- a/Classes/Domain/Service/DataHandler.php +++ b/Classes/Domain/Service/DataHandler.php @@ -83,16 +83,6 @@ class DataHandler implements Singleton $this->indexerFactory = $indexerFactory; } - /** - * @param string $table - * @param array $record - */ - public function add($table, array $record) - { - $this->logger->debug('Record received for add.', [$table, $record]); - $this->getIndexer($table)->indexDocument($record['uid']); - } - /** * @param string $table */ diff --git a/Classes/Hook/DataHandler.php b/Classes/Hook/DataHandler.php index d0eb1ba..fa4b09f 100644 --- a/Classes/Hook/DataHandler.php +++ b/Classes/Hook/DataHandler.php @@ -92,42 +92,36 @@ class DataHandler implements Singleton return true; } - /** - * Called by CoreDataHandler on database operations, e.g. if new records were created or records were updated. - * - * @param string $status - * @param string $table - * @param string|int $uid - * @param array $fieldArray - * @param CoreDataHandler $dataHandler - * - * @return bool False if hook was not processed. - */ - public function processDatamap_afterDatabaseOperations($status, $table, $uid, array $fieldArray, CoreDataHandler $dataHandler) + public function processDatamap_afterAllOperations(CoreDataHandler $dataHandler) + { + foreach ($dataHandler->datamap as $table => $record) { + $uid = key($record); + $fieldData = current($record); + + if (isset($fieldArray['uid'])) { + $uid = $fieldArray['uid']; + } elseif (isset($dataHandler->substNEWwithIDs[$uid])) { + $uid = $dataHandler->substNEWwithIDs[$uid]; + } + + $this->processRecord($table, $uid); + } + } + + protected function processRecord(string $table, int $uid) : bool { if (! $this->shouldProcessHookForTable($table)) { - $this->logger->debug('Database update not processed.', [$table, $uid]); + $this->logger->debug('Indexing of record not processed.', [$table, $uid]); return false; } - if ($status === 'new') { - $fieldArray['uid'] = $dataHandler->substNEWwithIDs[$uid]; - $this->dataHandler->add($table, $fieldArray); + $record = $this->getRecord($table, $uid); + if ($record !== null) { + $this->dataHandler->update($table, $record); return true; } - if ($status === 'update') { - $record = $this->getRecord($table, $uid); - if ($record !== null) { - $this->dataHandler->update($table, $record); - } - return true; - } - - $this->logger->debug( - 'Database update not processed, cause status is unhandled.', - [$status, $table, $uid, $fieldArray] - ); + $this->logger->debug('Indexing of record not processed, as he was not found in Database.', [$table, $uid]); return false; } diff --git a/Classes/Integration/Form/Finisher/DataHandlerFinisher.php b/Classes/Integration/Form/Finisher/DataHandlerFinisher.php index 02d6bde..696d8ad 100644 --- a/Classes/Integration/Form/Finisher/DataHandlerFinisher.php +++ b/Classes/Integration/Form/Finisher/DataHandlerFinisher.php @@ -58,10 +58,8 @@ class DataHandlerFinisher extends AbstractFinisher switch ($action) { case 'update': - $this->dataHandler->update($tableName, $record); - break; case 'add': - $this->dataHandler->add($tableName, $record); + $this->dataHandler->update($tableName, $record); break; case 'delete': $this->dataHandler->delete($tableName, $record['uid']); diff --git a/Tests/Functional/Hooks/DataHandler/IgnoresUnkownOperationTest.php b/Tests/Functional/Hooks/DataHandler/IgnoresUnkownOperationTest.php deleted file mode 100644 index b1676e3..0000000 --- a/Tests/Functional/Hooks/DataHandler/IgnoresUnkownOperationTest.php +++ /dev/null @@ -1,54 +0,0 @@ - - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; -use Codappix\SearchCore\Domain\Service\DataHandler as DataHandlerService; -use Codappix\SearchCore\Hook\DataHandler as DataHandlerHook; -use TYPO3\CMS\Core\DataHandling\DataHandler as Typo3DataHandler; -use TYPO3\CMS\Core\Utility\GeneralUtility; -use TYPO3\CMS\Extbase\Object\ObjectManager; - -class IgnoresUnkownOperationTest extends AbstractDataHandlerTest -{ - /** - * @var DataHandlerService|\PHPUnit_Framework_MockObject_MockObject|AccessibleObjectInterface - */ - protected $subject; - - /** - * @test - */ - public function dataHandlerCommandSomethingIsIgnored() - { - $subject = new DataHandlerHook($this->subject); - $this->assertFalse( - $subject->processDatamap_afterDatabaseOperations( - 'something', - 'tt_content', - 1, - [], - new Typo3DataHandler - ), - 'Hook processed status "something".' - ); - } -} diff --git a/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php b/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php index c33701d..509f053 100644 --- a/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php +++ b/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php @@ -64,7 +64,7 @@ class NonAllowedTablesTest extends AbstractDataHandlerTest /** * @test */ - public function updateWillNotBeTriggeredForSysCategory() + public function updateWillNotBeTriggeredForExistingSysCategory() { $this->subject->expects($this->exactly(0))->method('update'); @@ -83,9 +83,9 @@ class NonAllowedTablesTest extends AbstractDataHandlerTest /** * @test */ - public function addWillNotBeTriggeredForSysCategoy() + public function updateWillNotBeTriggeredForNewSysCategoy() { - $this->subject->expects($this->exactly(0))->method('add'); + $this->subject->expects($this->exactly(0))->method('update'); $tce = GeneralUtility::makeInstance(Typo3DataHandler::class); $tce->stripslashes_values = 0; diff --git a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php index 6a027cf..29ee14c 100644 --- a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php +++ b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php @@ -66,7 +66,7 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest /** * @test */ - public function updateWillBeTriggeredForTtContent() + public function updateWillBeTriggeredForExistingTtContent() { $this->subject->expects($this->exactly(1))->method('update') ->with( @@ -94,9 +94,9 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest /** * @test */ - public function addWillBeTriggeredForTtContent() + public function updateWillBeTriggeredForNewTtContent() { - $this->subject->expects($this->exactly(1))->method('add') + $this->subject->expects($this->exactly(1))->method('update') ->with( $this->equalTo('tt_content'), $this->callback(function ($record) { diff --git a/Tests/Unit/Integration/Form/Finisher/DataHandlerFinisherTest.php b/Tests/Unit/Integration/Form/Finisher/DataHandlerFinisherTest.php index 2211cc2..24c5059 100644 --- a/Tests/Unit/Integration/Form/Finisher/DataHandlerFinisherTest.php +++ b/Tests/Unit/Integration/Form/Finisher/DataHandlerFinisherTest.php @@ -83,19 +83,14 @@ class DataHandlerFinisherTest extends AbstractUnitTestCase public function possibleFinisherSetup() : array { return [ - 'valid add configuration' => [ - 'action' => 'add', - 'nonCalledActions' => ['delete', 'update'], - 'expectedSecondArgument' => ['uid' => 23], - ], 'valid update configuration' => [ 'action' => 'update', - 'nonCalledActions' => ['delete', 'add'], + 'nonCalledActions' => ['delete'], 'expectedSecondArgument' => ['uid' => 23], ], 'valid delete configuration' => [ 'action' => 'delete', - 'nonCalledActions' => ['update', 'add'], + 'nonCalledActions' => ['update'], 'expectedSecondArgument' => 23, ], ]; @@ -109,7 +104,7 @@ class DataHandlerFinisherTest extends AbstractUnitTestCase { $this->subject->setOptions($options); - foreach (['add', 'update', 'delete'] as $nonCalledAction) { + foreach (['update', 'delete'] as $nonCalledAction) { $this->dataHandlerMock->expects($this->never())->method($nonCalledAction); } From 350f8a52b621500433f3a776fb6bf03c34c1a96b Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 20 Feb 2018 12:01:28 +0100 Subject: [PATCH 09/75] FEATURE: Use extbase for processor instantiation This way injects will be resolved inside of processors, enabling developers to inject dependencies. We use inject instead of constructor as indexers mostly will change the constructor and should not need to add the objectmanager. Resolves: #117 --- Classes/Domain/Index/AbstractIndexer.php | 9 ++++++++- Documentation/source/configuration/indexing.rst | 3 +++ Tests/Unit/Domain/Index/AbstractIndexerTest.php | 14 ++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/Classes/Domain/Index/AbstractIndexer.php b/Classes/Domain/Index/AbstractIndexer.php index bc8674e..c48ecb6 100644 --- a/Classes/Domain/Index/AbstractIndexer.php +++ b/Classes/Domain/Index/AbstractIndexer.php @@ -48,6 +48,12 @@ abstract class AbstractIndexer implements IndexerInterface */ protected $logger; + /** + * @var \TYPO3\CMS\Extbase\Object\ObjectManagerInterface + * @inject + */ + protected $objectManager; + /** * Inject log manager to get concrete logger from it. * @@ -141,7 +147,8 @@ abstract class AbstractIndexer implements IndexerInterface } else { $className = $configuration['_typoScriptNodeValue']; } - $dataProcessor = GeneralUtility::makeInstance($className); + + $dataProcessor = $this->objectManager->get($className); if ($dataProcessor instanceof ProcessorInterface) { $record = $dataProcessor->processRecord($record, $configuration); } diff --git a/Documentation/source/configuration/indexing.rst b/Documentation/source/configuration/indexing.rst index b605008..619e56a 100644 --- a/Documentation/source/configuration/indexing.rst +++ b/Documentation/source/configuration/indexing.rst @@ -205,3 +205,6 @@ class name) as done in the examples above. By implementing also the same interface as necessary for TYPO3 :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`, you are able to reuse the same code also for Fluid to prepare the same record fetched from DB for your fluid. + +Dependency injection is possible inside of processors, as we instantiate through extbase +``ObjectManager``. diff --git a/Tests/Unit/Domain/Index/AbstractIndexerTest.php b/Tests/Unit/Domain/Index/AbstractIndexerTest.php index 3bbc97d..8c1e527 100644 --- a/Tests/Unit/Domain/Index/AbstractIndexerTest.php +++ b/Tests/Unit/Domain/Index/AbstractIndexerTest.php @@ -26,6 +26,7 @@ use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\DataProcessing\CopyToProcessor; use Codappix\SearchCore\Domain\Index\AbstractIndexer; use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; +use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; class AbstractIndexerTest extends AbstractUnitTestCase { @@ -44,17 +45,25 @@ class AbstractIndexerTest extends AbstractUnitTestCase */ protected $connection; + /** + * @var ObjectManagerInterface + */ + protected $objectManager; + public function setUp() { parent::setUp(); $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock(); $this->connection = $this->getMockBuilder(ConnectionInterface::class)->getMock(); + $this->objectManager = $this->getMockBuilder(ObjectManagerInterface::class)->getMock(); + $this->subject = $this->getMockForAbstractClass(AbstractIndexer::class, [ $this->connection, $this->configuration ]); + $this->inject($this->subject, 'objectManager', $this->objectManager); $this->subject->injectLogger($this->getMockedLogger()); $this->subject->setIdentifier('testTable'); $this->subject->expects($this->any()) @@ -73,6 +82,11 @@ class AbstractIndexerTest extends AbstractUnitTestCase $expectedRecord['new_test_field2'] = 'test' . PHP_EOL . 'test'; $expectedRecord['search_abstract'] = ''; + $this->objectManager->expects($this->any()) + ->method('get') + ->with(CopyToProcessor::class) + ->willReturn(new CopyToProcessor()); + $this->configuration->expects($this->exactly(2)) ->method('get') ->withConsecutive(['indexing.testTable.dataProcessing'], ['indexing.testTable.abstractFields']) From 2998c43ba824492bbb888e6b99c1c0f349c7e5ad Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 1 Mar 2018 08:03:51 +0100 Subject: [PATCH 10/75] TASK: Refactor data processing Use own service to handle data processing. Classes like indexer should not know anything about the structure and how to process the data. Also rename record to data, as we can process just any data in form of an array. Relates: #116 --- Classes/DataProcessing/CopyToProcessor.php | 2 +- Classes/DataProcessing/GeoPointProcessor.php | 2 +- Classes/DataProcessing/ProcessorInterface.php | 7 +-- Classes/DataProcessing/RemoveProcessor.php | 2 +- Classes/DataProcessing/Service.php | 56 +++++++++++++++++++ Classes/Domain/Index/AbstractIndexer.php | 18 +++--- .../DataProcessing/CopyToProcessorTest.php | 16 +++--- .../DataProcessing/GeoPointProcessorTest.php | 20 +++---- .../DataProcessing/RemoveProcessorTest.php | 22 ++++---- 9 files changed, 98 insertions(+), 47 deletions(-) create mode 100644 Classes/DataProcessing/Service.php diff --git a/Classes/DataProcessing/CopyToProcessor.php b/Classes/DataProcessing/CopyToProcessor.php index eea555f..28c4294 100644 --- a/Classes/DataProcessing/CopyToProcessor.php +++ b/Classes/DataProcessing/CopyToProcessor.php @@ -25,7 +25,7 @@ namespace Codappix\SearchCore\DataProcessing; */ class CopyToProcessor implements ProcessorInterface { - public function processRecord(array $record, array $configuration) : array + public function processData(array $record, array $configuration) : array { $all = []; diff --git a/Classes/DataProcessing/GeoPointProcessor.php b/Classes/DataProcessing/GeoPointProcessor.php index f9256b3..c5b6d18 100644 --- a/Classes/DataProcessing/GeoPointProcessor.php +++ b/Classes/DataProcessing/GeoPointProcessor.php @@ -25,7 +25,7 @@ namespace Codappix\SearchCore\DataProcessing; */ class GeoPointProcessor implements ProcessorInterface { - public function processRecord(array $record, array $configuration) : array + public function processData(array $record, array $configuration) : array { if (! $this->canApply($record, $configuration)) { return $record; diff --git a/Classes/DataProcessing/ProcessorInterface.php b/Classes/DataProcessing/ProcessorInterface.php index 4dc0794..f7513f2 100644 --- a/Classes/DataProcessing/ProcessorInterface.php +++ b/Classes/DataProcessing/ProcessorInterface.php @@ -21,14 +21,13 @@ namespace Codappix\SearchCore\DataProcessing; */ /** - * All DataProcessing Processors should implement this interface, otherwise they - * will not be executed. + * All DataProcessing Processors should implement this interface. */ interface ProcessorInterface { /** - * Processes the given record. + * Processes the given data. * Also retrieves the configuration for this processor instance. */ - public function processRecord(array $record, array $configuration) : array; + public function processData(array $record, array $configuration) : array; } diff --git a/Classes/DataProcessing/RemoveProcessor.php b/Classes/DataProcessing/RemoveProcessor.php index 6e74237..b8d6283 100644 --- a/Classes/DataProcessing/RemoveProcessor.php +++ b/Classes/DataProcessing/RemoveProcessor.php @@ -27,7 +27,7 @@ use TYPO3\CMS\Core\Utility\GeneralUtility; */ class RemoveProcessor implements ProcessorInterface { - public function processRecord(array $record, array $configuration) : array + public function processData(array $record, array $configuration) : array { if (!isset($configuration['fields'])) { return $record; diff --git a/Classes/DataProcessing/Service.php b/Classes/DataProcessing/Service.php new file mode 100644 index 0000000..3e3b053 --- /dev/null +++ b/Classes/DataProcessing/Service.php @@ -0,0 +1,56 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; + +/** + * Eases work with data processing. + */ +class Service +{ + /** + * @var ObjectManagerInterface + */ + protected $objectManager; + + public function __construct(ObjectManagerInterface $objectManager) + { + $this->objectManager = $objectManager; + } + + /** + * Executes the dataprocessor depending on configuration and returns the result. + * + * @param array|string $configuration Either the full configuration or only the class name. + */ + public function executeDataProcessor($configuration, array $data) : array + { + if (is_string($configuration)) { + $configuration = [ + '_typoScriptNodeValue' => $configuration, + ]; + } + + return $this->objectManager->get($configuration['_typoScriptNodeValue']) + ->processData($data, $configuration); + } +} diff --git a/Classes/Domain/Index/AbstractIndexer.php b/Classes/Domain/Index/AbstractIndexer.php index bc8674e..2de7dd5 100644 --- a/Classes/Domain/Index/AbstractIndexer.php +++ b/Classes/Domain/Index/AbstractIndexer.php @@ -43,6 +43,12 @@ abstract class AbstractIndexer implements IndexerInterface */ protected $identifier = ''; + /** + * @var \Codappix\SearchCore\DataProcessing\Service + * @inject + */ + protected $dataProcessorService; + /** * @var \TYPO3\CMS\Core\Log\Logger */ @@ -134,17 +140,7 @@ abstract class AbstractIndexer implements IndexerInterface { try { foreach ($this->configuration->get('indexing.' . $this->identifier . '.dataProcessing') as $configuration) { - $className = ''; - if (is_string($configuration)) { - $className = $configuration; - $configuration = []; - } else { - $className = $configuration['_typoScriptNodeValue']; - } - $dataProcessor = GeneralUtility::makeInstance($className); - if ($dataProcessor instanceof ProcessorInterface) { - $record = $dataProcessor->processRecord($record, $configuration); - } + $record = $this->dataProcessorService->executeDataProcessor($configuration, $record); } } catch (InvalidArgumentException $e) { // Nothing to do. diff --git a/Tests/Unit/DataProcessing/CopyToProcessorTest.php b/Tests/Unit/DataProcessing/CopyToProcessorTest.php index 2f9e498..28545a3 100644 --- a/Tests/Unit/DataProcessing/CopyToProcessorTest.php +++ b/Tests/Unit/DataProcessing/CopyToProcessorTest.php @@ -27,15 +27,15 @@ class CopyToProcessorTest extends AbstractUnitTestCase { /** * @test - * @dataProvider getPossibleRecordConfigurationCombinations + * @dataProvider getPossibleDataConfigurationCombinations */ - public function fieldsAreCopiedAsConfigured(array $record, array $configuration, array $expectedRecord) + public function fieldsAreCopiedAsConfigured(array $record, array $configuration, array $expectedData) { $subject = new CopyToProcessor(); - $processedRecord = $subject->processRecord($record, $configuration); + $processedData = $subject->processData($record, $configuration); $this->assertSame( - $expectedRecord, - $processedRecord, + $expectedData, + $processedData, 'The processor did not return the expected processed record.' ); } @@ -43,7 +43,7 @@ class CopyToProcessorTest extends AbstractUnitTestCase /** * @return array */ - public function getPossibleRecordConfigurationCombinations() + public function getPossibleDataConfigurationCombinations() { return [ 'Copy all fields to new field' => [ @@ -54,7 +54,7 @@ class CopyToProcessorTest extends AbstractUnitTestCase 'configuration' => [ 'to' => 'new_field', ], - 'expectedRecord' => [ + 'expectedData' => [ 'field 1' => 'Some content like lorem', 'field 2' => 'Some more content like ipsum', 'new_field' => 'Some content like lorem' . PHP_EOL . 'Some more content like ipsum', @@ -71,7 +71,7 @@ class CopyToProcessorTest extends AbstractUnitTestCase 'configuration' => [ 'to' => 'new_field', ], - 'expectedRecord' => [ + 'expectedData' => [ 'field 1' => 'Some content like lorem', 'field with sub2' => [ 'Tag 1', diff --git a/Tests/Unit/DataProcessing/GeoPointProcessorTest.php b/Tests/Unit/DataProcessing/GeoPointProcessorTest.php index 99b8eb3..02db659 100644 --- a/Tests/Unit/DataProcessing/GeoPointProcessorTest.php +++ b/Tests/Unit/DataProcessing/GeoPointProcessorTest.php @@ -27,15 +27,15 @@ class GeoPointProcessorTest extends AbstractUnitTestCase { /** * @test - * @dataProvider getPossibleRecordConfigurationCombinations + * @dataProvider getPossibleDataConfigurationCombinations */ - public function geoPointsAreAddedAsConfigured(array $record, array $configuration, array $expectedRecord) + public function geoPointsAreAddedAsConfigured(array $record, array $configuration, array $expectedData) { $subject = new GeoPointProcessor(); - $processedRecord = $subject->processRecord($record, $configuration); + $processedData = $subject->processData($record, $configuration); $this->assertSame( - $expectedRecord, - $processedRecord, + $expectedData, + $processedData, 'The processor did not return the expected processed record.' ); } @@ -43,7 +43,7 @@ class GeoPointProcessorTest extends AbstractUnitTestCase /** * @return array */ - public function getPossibleRecordConfigurationCombinations() + public function getPossibleDataConfigurationCombinations() { return [ 'Create new field with existing lat and lng' => [ @@ -56,7 +56,7 @@ class GeoPointProcessorTest extends AbstractUnitTestCase 'lat' => 'lat', 'lon' => 'lng', ], - 'expectedRecord' => [ + 'expectedData' => [ 'lat' => 23.232, 'lng' => 45.43, 'location' => [ @@ -73,7 +73,7 @@ class GeoPointProcessorTest extends AbstractUnitTestCase 'configuration' => [ 'to' => 'location', ], - 'expectedRecord' => [ + 'expectedData' => [ 'lat' => 23.232, 'lng' => 45.43, ], @@ -88,7 +88,7 @@ class GeoPointProcessorTest extends AbstractUnitTestCase 'lat' => 'lat', 'lon' => 'lng', ], - 'expectedRecord' => [ + 'expectedData' => [ 'lat' => '', 'lng' => '', ], @@ -103,7 +103,7 @@ class GeoPointProcessorTest extends AbstractUnitTestCase 'lat' => 'lat', 'lon' => 'lng', ], - 'expectedRecord' => [ + 'expectedData' => [ 'lat' => 'av', 'lng' => 'dsf', ], diff --git a/Tests/Unit/DataProcessing/RemoveProcessorTest.php b/Tests/Unit/DataProcessing/RemoveProcessorTest.php index dc55f73..cd23d16 100644 --- a/Tests/Unit/DataProcessing/RemoveProcessorTest.php +++ b/Tests/Unit/DataProcessing/RemoveProcessorTest.php @@ -27,15 +27,15 @@ class RemoveProcessorTest extends AbstractUnitTestCase { /** * @test - * @dataProvider getPossibleRecordConfigurationCombinations + * @dataProvider getPossibleDataConfigurationCombinations */ - public function fieldsAreCopiedAsConfigured(array $record, array $configuration, array $expectedRecord) + public function fieldsAreCopiedAsConfigured(array $record, array $configuration, array $expectedData) { $subject = new RemoveProcessor(); - $processedRecord = $subject->processRecord($record, $configuration); + $processedData = $subject->processData($record, $configuration); $this->assertSame( - $expectedRecord, - $processedRecord, + $expectedData, + $processedData, 'The processor did not return the expected processed record.' ); } @@ -43,7 +43,7 @@ class RemoveProcessorTest extends AbstractUnitTestCase /** * @return array */ - public function getPossibleRecordConfigurationCombinations() + public function getPossibleDataConfigurationCombinations() { return [ 'Nothing configured' => [ @@ -56,7 +56,7 @@ class RemoveProcessorTest extends AbstractUnitTestCase ], 'configuration' => [ ], - 'expectedRecord' => [ + 'expectedData' => [ 'field 1' => 'Some content like lorem', 'field with sub2' => [ 'Tag 1', @@ -76,7 +76,7 @@ class RemoveProcessorTest extends AbstractUnitTestCase 'fields' => 'field with sub2', '_typoScriptNodeValue' => 'Codappix\SearchCore\DataProcessing\RemoveProcessor', ], - 'expectedRecord' => [ + 'expectedData' => [ 'field 1' => 'Some content like lorem', ], ], @@ -92,7 +92,7 @@ class RemoveProcessorTest extends AbstractUnitTestCase 'fields' => 'non existing', '_typoScriptNodeValue' => 'Codappix\SearchCore\DataProcessing\RemoveProcessor', ], - 'expectedRecord' => [ + 'expectedData' => [ 'field 1' => 'Some content like lorem', 'field with sub2' => [ 'Tag 1', @@ -113,7 +113,7 @@ class RemoveProcessorTest extends AbstractUnitTestCase 'fields' => 'field 3, field with sub2', '_typoScriptNodeValue' => 'Codappix\SearchCore\DataProcessing\RemoveProcessor', ], - 'expectedRecord' => [ + 'expectedData' => [ 'field 1' => 'Some content like lorem', ], ], @@ -125,7 +125,7 @@ class RemoveProcessorTest extends AbstractUnitTestCase 'fields' => 'field 1', '_typoScriptNodeValue' => 'Codappix\SearchCore\DataProcessing\RemoveProcessor', ], - 'expectedRecord' => [ + 'expectedData' => [ ], ], ]; From 769fb8237b14671f8e50717c100b39a7f06dc97c Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 09:04:47 +0100 Subject: [PATCH 11/75] TASK: Add data processing to search result Search results are now processed through data processing by search service. The result will be a SearchResult model from our domain. Also SearchResult will execute new queries, e.g. from paginate widget, through SearchService to apply data processing again. Remove duplicate stub code to trait, to keep own logic and code clean. --- .../Connection/Elasticsearch/SearchResult.php | 39 +----- Classes/Connection/ResultItemInterface.php | 9 +- Classes/Connection/SearchRequestInterface.php | 7 + .../Domain/Model/QueryResultInterfaceStub.php | 61 +++++++++ .../Model}/ResultItem.php | 11 +- Classes/Domain/Model/SearchRequest.php | 13 +- Classes/Domain/Model/SearchResult.php | 129 ++++++++++++++++++ Classes/Domain/Search/SearchService.php | 46 ++++++- .../Unit/Domain/Index/AbstractIndexerTest.php | 35 ++++- .../Unit/Domain/Search/SearchServiceTest.php | 84 +++++++++--- 10 files changed, 369 insertions(+), 65 deletions(-) create mode 100644 Classes/Domain/Model/QueryResultInterfaceStub.php rename Classes/{Connection/Elasticsearch => Domain/Model}/ResultItem.php (88%) create mode 100644 Classes/Domain/Model/SearchResult.php diff --git a/Classes/Connection/Elasticsearch/SearchResult.php b/Classes/Connection/Elasticsearch/SearchResult.php index 3919722..867d823 100644 --- a/Classes/Connection/Elasticsearch/SearchResult.php +++ b/Classes/Connection/Elasticsearch/SearchResult.php @@ -24,10 +24,14 @@ use Codappix\SearchCore\Connection\FacetInterface; use Codappix\SearchCore\Connection\ResultItemInterface; use Codappix\SearchCore\Connection\SearchRequestInterface; use Codappix\SearchCore\Connection\SearchResultInterface; +use Codappix\SearchCore\Domain\Model\QueryResultInterfaceStub; +use Codappix\SearchCore\Domain\Model\ResultItem; use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; class SearchResult implements SearchResultInterface { + use QueryResultInterfaceStub; + /** * @var SearchRequestInterface */ @@ -104,7 +108,7 @@ class SearchResult implements SearchResultInterface } foreach ($this->result->getResults() as $result) { - $this->results[] = new ResultItem($result); + $this->results[] = new ResultItem($result->getData()); } } @@ -153,41 +157,8 @@ class SearchResult implements SearchResultInterface $this->position = 0; } - // Extbase QueryResultInterface - Implemented to support Pagination of Fluid. - public function getQuery() { return $this->searchRequest; } - - public function getFirst() - { - throw new \BadMethodCallException('Method is not implemented yet.', 1502195121); - } - - public function toArray() - { - throw new \BadMethodCallException('Method is not implemented yet.', 1502195135); - } - - public function offsetExists($offset) - { - // Return false to allow Fluid to use appropriate getter methods. - return false; - } - - public function offsetGet($offset) - { - throw new \BadMethodCallException('Use getter to fetch properties.', 1502196933); - } - - public function offsetSet($offset, $value) - { - throw new \BadMethodCallException('You are not allowed to modify the result.', 1502196934); - } - - public function offsetUnset($offset) - { - throw new \BadMethodCallException('You are not allowed to modify the result.', 1502196936); - } } diff --git a/Classes/Connection/ResultItemInterface.php b/Classes/Connection/ResultItemInterface.php index 56add2a..d7fe1a9 100644 --- a/Classes/Connection/ResultItemInterface.php +++ b/Classes/Connection/ResultItemInterface.php @@ -25,5 +25,12 @@ namespace Codappix\SearchCore\Connection; */ interface ResultItemInterface extends \ArrayAccess { - + /** + * Returns every information as array. + * + * Provide key/column/field => data. + * + * Used e.g. for dataprocessing. + */ + public function getPlainData() : array; } diff --git a/Classes/Connection/SearchRequestInterface.php b/Classes/Connection/SearchRequestInterface.php index 7c7956e..e24adc3 100644 --- a/Classes/Connection/SearchRequestInterface.php +++ b/Classes/Connection/SearchRequestInterface.php @@ -20,6 +20,7 @@ namespace Codappix\SearchCore\Connection; * 02110-1301, USA. */ +use Codappix\SearchCore\Domain\Search\SearchService; use TYPO3\CMS\Extbase\Persistence\QueryInterface; interface SearchRequestInterface extends QueryInterface @@ -40,4 +41,10 @@ interface SearchRequestInterface extends QueryInterface * @return array */ public function getFilter(); + + /** + * Workaround for paginate widget support which will + * use the request to build another search. + */ + public function setSearchService(SearchService $searchService); } diff --git a/Classes/Domain/Model/QueryResultInterfaceStub.php b/Classes/Domain/Model/QueryResultInterfaceStub.php new file mode 100644 index 0000000..960fd40 --- /dev/null +++ b/Classes/Domain/Model/QueryResultInterfaceStub.php @@ -0,0 +1,61 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * As we have to stay compatible with QueryResultInterface + * of extbase but can and need not to provide all methods, + * this stub will provde the non implemented methods to + * keep real implementations clean. + */ +trait QueryResultInterfaceStub +{ + public function getFirst() + { + throw new \BadMethodCallException('Method is not implemented yet.', 1502195121); + } + + public function toArray() + { + throw new \BadMethodCallException('Method is not implemented yet.', 1502195135); + } + + public function offsetExists($offset) + { + // Return false to allow Fluid to use appropriate getter methods. + return false; + } + + public function offsetGet($offset) + { + throw new \BadMethodCallException('Use getter to fetch properties.', 1502196933); + } + + public function offsetSet($offset, $value) + { + throw new \BadMethodCallException('You are not allowed to modify the result.', 1502196934); + } + + public function offsetUnset($offset) + { + throw new \BadMethodCallException('You are not allowed to modify the result.', 1502196936); + } +} diff --git a/Classes/Connection/Elasticsearch/ResultItem.php b/Classes/Domain/Model/ResultItem.php similarity index 88% rename from Classes/Connection/Elasticsearch/ResultItem.php rename to Classes/Domain/Model/ResultItem.php index e56783c..f63b408 100644 --- a/Classes/Connection/Elasticsearch/ResultItem.php +++ b/Classes/Domain/Model/ResultItem.php @@ -1,5 +1,5 @@ @@ -29,9 +29,14 @@ class ResultItem implements ResultItemInterface */ protected $data = []; - public function __construct(\Elastica\Result $result) + public function __construct(array $result) { - $this->data = $result->getData(); + $this->data = $data; + } + + public function getPlainData() : array + { + return $this->data; } public function offsetExists($offset) diff --git a/Classes/Domain/Model/SearchRequest.php b/Classes/Domain/Model/SearchRequest.php index d751ce8..d835e7b 100644 --- a/Classes/Domain/Model/SearchRequest.php +++ b/Classes/Domain/Model/SearchRequest.php @@ -23,6 +23,7 @@ namespace Codappix\SearchCore\Domain\Model; use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Connection\FacetRequestInterface; use Codappix\SearchCore\Connection\SearchRequestInterface; +use Codappix\SearchCore\Domain\Search\SearchService; /** * Represents a search request used to process an actual search. @@ -63,6 +64,11 @@ class SearchRequest implements SearchRequestInterface */ protected $connection = null; + /** + * @var SearchService + */ + protected $searchService = null; + /** * @param string $query */ @@ -143,12 +149,17 @@ class SearchRequest implements SearchRequestInterface $this->connection = $connection; } + public function setSearchService(SearchService $searchService) + { + $this->searchService = $searchService; + } + // Extbase QueryInterface // Current implementation covers only paginate widget support. public function execute($returnRawQueryResult = false) { if ($this->connection instanceof ConnectionInterface) { - return $this->connection->search($this); + return $this->searchService->processResult($this->connection->search($this)); } throw new \InvalidArgumentException( diff --git a/Classes/Domain/Model/SearchResult.php b/Classes/Domain/Model/SearchResult.php new file mode 100644 index 0000000..d91820d --- /dev/null +++ b/Classes/Domain/Model/SearchResult.php @@ -0,0 +1,129 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Connection\ResultItemInterface; +use Codappix\SearchCore\Connection\SearchResultInterface; +use Codappix\SearchCore\Domain\Model\QueryResultInterfaceStub; + +/** + * Generic model for mapping a concrete search result from a connection. + */ +class SearchResult implements SearchResultInterface +{ + use QueryResultInterfaceStub; + + /** + * @var SearchResultInterface + */ + protected $originalSearchResult; + + /** + * @var array + */ + protected $resultItems = []; + + /** + * @var array + */ + protected $results = []; + + /** + * For Iterator interface. + * + * @var int + */ + protected $position = 0; + + public function __construct(SearchResultInterface $originalSearchResult, array $resultItems) + { + $this->originalSearchResult = $originalSearchResult; + $this->resultItems = $resultItems; + } + + /** + * @return array + */ + public function getResults() + { + $this->initResults(); + + return $this->results; + } + + protected function initResults() + { + if ($this->results !== []) { + return; + } + + foreach ($this->resultItems as $item) { + $this->results[] = new ResultItem($item); + } + } + + public function getFacets() + { + return $this->originalSearchResult->getFacets(); + } + + public function getCurrentCount() + { + return $this->originalSearchResult->getCurrentCount(); + } + + public function count() + { + return $this->originalSearchResult->count(); + } + + public function current() + { + return $this->getResults()[$this->position]; + } + + public function next() + { + ++$this->position; + + return $this->current(); + } + + public function key() + { + return $this->position; + } + + public function valid() + { + return isset($this->getResults()[$this->position]); + } + + public function rewind() + { + $this->position = 0; + } + + public function getQuery() + { + return $this->originalSearchResult->getQuery(); + } +} diff --git a/Classes/Domain/Search/SearchService.php b/Classes/Domain/Search/SearchService.php index 120ab3b..48be32c 100644 --- a/Classes/Domain/Search/SearchService.php +++ b/Classes/Domain/Search/SearchService.php @@ -25,7 +25,9 @@ use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Connection\SearchRequestInterface; use Codappix\SearchCore\Connection\SearchResultInterface; +use Codappix\SearchCore\DataProcessing\Service as DataProcessorService; use Codappix\SearchCore\Domain\Model\FacetRequest; +use Codappix\SearchCore\Domain\Model\SearchResult; use TYPO3\CMS\Core\Utility\ArrayUtility; use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; @@ -49,19 +51,27 @@ class SearchService */ protected $objectManager; + /** + * @var DataProcessorService + */ + protected $dataProcessorService; + /** * @param ConnectionInterface $connection * @param ConfigurationContainerInterface $configuration * @param ObjectManagerInterface $objectManager + * @param DataProcessorService $dataProcessorService */ public function __construct( ConnectionInterface $connection, ConfigurationContainerInterface $configuration, - ObjectManagerInterface $objectManager + ObjectManagerInterface $objectManager, + DataProcessorService $dataProcessorService ) { $this->connection = $connection; $this->configuration = $configuration; $this->objectManager = $objectManager; + $this->dataProcessorService = $dataProcessorService; } /** @@ -70,12 +80,15 @@ class SearchService */ public function search(SearchRequestInterface $searchRequest) { - $searchRequest->setConnection($this->connection); $this->addSize($searchRequest); $this->addConfiguredFacets($searchRequest); $this->addConfiguredFilters($searchRequest); - return $this->connection->search($searchRequest); + // Add connection to request to enable paginate widget support + $searchRequest->setConnection($this->connection); + $searchRequest->setSearchService($this); + + return $this->processResult($this->connection->search($searchRequest)); } /** @@ -138,4 +151,31 @@ class SearchService // Nothing todo, no filter configured. } } + + /** + * Processes the result, e.g. applies configured data processing to result. + */ + public function processResult(SearchResultInterface $searchResult) : SearchResultInterface + { + try { + $newSearchResultItems = []; + foreach ($this->configuration->get('searching.dataProcessing') as $configuration) { + foreach ($searchResult as $resultItem) { + $newSearchResultItems[] = $this->dataProcessorService->executeDataProcessor( + $configuration, + $resultItem->getPlainData() + ); + } + } + + return $this->objectManager->get( + SearchResult::class, + $searchResult, + $newSearchResultItems + ); + } + catch (InvalidArgumentException $e) { + return $searchResult; + } + } } diff --git a/Tests/Unit/Domain/Index/AbstractIndexerTest.php b/Tests/Unit/Domain/Index/AbstractIndexerTest.php index 3bbc97d..411cc24 100644 --- a/Tests/Unit/Domain/Index/AbstractIndexerTest.php +++ b/Tests/Unit/Domain/Index/AbstractIndexerTest.php @@ -24,6 +24,7 @@ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\DataProcessing\CopyToProcessor; +use Codappix\SearchCore\DataProcessing\Service as DataProcessorService; use Codappix\SearchCore\Domain\Index\AbstractIndexer; use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; @@ -44,17 +45,26 @@ class AbstractIndexerTest extends AbstractUnitTestCase */ protected $connection; + /** + * @var DataProcessorService + */ + protected $dataProcessorService; + public function setUp() { parent::setUp(); $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock(); $this->connection = $this->getMockBuilder(ConnectionInterface::class)->getMock(); + $this->dataProcessorService = $this->getMockBuilder(DataProcessorService::class) + ->disableOriginalConstructor() + ->getMock(); $this->subject = $this->getMockForAbstractClass(AbstractIndexer::class, [ $this->connection, $this->configuration ]); + $this->inject($this->subject, 'dataProcessorService', $this->dataProcessorService); $this->subject->injectLogger($this->getMockedLogger()); $this->subject->setIdentifier('testTable'); $this->subject->expects($this->any()) @@ -73,7 +83,30 @@ class AbstractIndexerTest extends AbstractUnitTestCase $expectedRecord['new_test_field2'] = 'test' . PHP_EOL . 'test'; $expectedRecord['search_abstract'] = ''; - $this->configuration->expects($this->exactly(2)) + $this->dataProcessorService->expects($this->any()) + ->method('executeDataProcessor') + ->withConsecutive( + [ + [ + '_typoScriptNodeValue' => CopyToProcessor::class, + 'to' => 'new_test_field', + ], + $record, + ], + [ + [ + '_typoScriptNodeValue' => CopyToProcessor::class, + 'to' => 'new_test_field2', + ], + array_merge($record, ['new_test_field' => 'test']), + ] + ) + ->will($this->onConsecutiveCalls( + array_merge($record, ['new_test_field' => 'test']), + $expectedRecord + )); + + $this->configuration->expects($this->any()) ->method('get') ->withConsecutive(['indexing.testTable.dataProcessing'], ['indexing.testTable.abstractFields']) ->will($this->onConsecutiveCalls([ diff --git a/Tests/Unit/Domain/Search/SearchServiceTest.php b/Tests/Unit/Domain/Search/SearchServiceTest.php index c63e29e..aaf2e5c 100644 --- a/Tests/Unit/Domain/Search/SearchServiceTest.php +++ b/Tests/Unit/Domain/Search/SearchServiceTest.php @@ -23,6 +23,8 @@ namespace Copyright\SearchCore\Tests\Unit\Domain\Search; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Connection\ConnectionInterface; +use Codappix\SearchCore\Connection\SearchResultInterface; +use Codappix\SearchCore\DataProcessing\Service as DataProcessorService; use Codappix\SearchCore\Domain\Model\SearchRequest; use Codappix\SearchCore\Domain\Search\SearchService; use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; @@ -35,24 +37,59 @@ class SearchServiceTest extends AbstractUnitTestCase */ protected $subject; + /** + * @var SearchResultInterface + */ + protected $result; + + /** + * @var ConnectionInterface + */ + protected $connection; + + /** + * @var ConfigurationContainerInterface + */ + protected $configuration; + + /** + * @var ObjectManagerInterface + */ + protected $objectManager; + + /** + * @var DataProcessorService + */ + protected $dataProcessorService; + public function setUp() { parent::setUp(); + $this->result = $this->getMockBuilder(SearchResultInterface::class) + ->disableOriginalConstructor() + ->getMock(); $this->connection = $this->getMockBuilder(ConnectionInterface::class) ->disableOriginalConstructor() ->getMock(); + $this->connection->expects($this->any()) + ->method('search') + ->willReturn($this->result); $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class) ->disableOriginalConstructor() ->getMock(); $this->objectManager = $this->getMockBuilder(ObjectManagerInterface::class) ->disableOriginalConstructor() ->getMock(); + $this->dataProcessorService = $this->getMockBuilder(DataProcessorService::class) + ->setConstructorArgs([$this->objectManager]) + ->getMock(); $this->subject = new SearchService( $this->connection, $this->configuration, - $this->objectManager + $this->objectManager, + $this->dataProcessorService ); } @@ -61,13 +98,12 @@ class SearchServiceTest extends AbstractUnitTestCase */ public function sizeIsAddedFromConfiguration() { - $this->configuration->expects($this->exactly(2)) + $this->configuration->expects($this->any()) ->method('getIfExists') ->withConsecutive(['searching.size'], ['searching.facets']) ->will($this->onConsecutiveCalls(45, null)); - $this->configuration->expects($this->exactly(1)) + $this->configuration->expects($this->any()) ->method('get') - ->with('searching.filter') ->will($this->throwException(new InvalidArgumentException)); $this->connection->expects($this->once()) ->method('search') @@ -84,13 +120,12 @@ class SearchServiceTest extends AbstractUnitTestCase */ public function defaultSizeIsAddedIfNothingIsConfigured() { - $this->configuration->expects($this->exactly(2)) + $this->configuration->expects($this->any()) ->method('getIfExists') ->withConsecutive(['searching.size'], ['searching.facets']) ->will($this->onConsecutiveCalls(null, null)); - $this->configuration->expects($this->exactly(1)) + $this->configuration->expects($this->any()) ->method('get') - ->with('searching.filter') ->will($this->throwException(new InvalidArgumentException)); $this->connection->expects($this->once()) ->method('search') @@ -107,14 +142,16 @@ class SearchServiceTest extends AbstractUnitTestCase */ public function configuredFilterAreAddedToRequestWithoutAnyFilter() { - $this->configuration->expects($this->exactly(2)) + $this->configuration->expects($this->any()) ->method('getIfExists') ->withConsecutive(['searching.size'], ['searching.facets']) ->will($this->onConsecutiveCalls(null, null)); - $this->configuration->expects($this->exactly(1)) + $this->configuration->expects($this->any()) ->method('get') - ->with('searching.filter') - ->willReturn(['property' => 'something']); + ->will($this->onConsecutiveCalls( + ['property' => 'something'], + $this->throwException(new InvalidArgumentException) + )); $this->connection->expects($this->once()) ->method('search') @@ -131,14 +168,16 @@ class SearchServiceTest extends AbstractUnitTestCase */ public function configuredFilterAreAddedToRequestWithExistingFilter() { - $this->configuration->expects($this->exactly(2)) + $this->configuration->expects($this->any()) ->method('getIfExists') ->withConsecutive(['searching.size'], ['searching.facets']) ->will($this->onConsecutiveCalls(null, null)); - $this->configuration->expects($this->exactly(1)) + $this->configuration->expects($this->any()) ->method('get') - ->with('searching.filter') - ->willReturn(['property' => 'something']); + ->will($this->onConsecutiveCalls( + ['property' => 'something'], + $this->throwException(new InvalidArgumentException) + )); $this->connection->expects($this->once()) ->method('search') @@ -159,13 +198,12 @@ class SearchServiceTest extends AbstractUnitTestCase */ public function nonConfiguredFilterIsNotChangingRequestWithExistingFilter() { - $this->configuration->expects($this->exactly(2)) + $this->configuration->expects($this->any()) ->method('getIfExists') ->withConsecutive(['searching.size'], ['searching.facets']) ->will($this->onConsecutiveCalls(null, null)); - $this->configuration->expects($this->exactly(1)) + $this->configuration->expects($this->any()) ->method('get') - ->with('searching.filter') ->will($this->throwException(new InvalidArgumentException)); $this->connection->expects($this->once()) @@ -184,14 +222,16 @@ class SearchServiceTest extends AbstractUnitTestCase */ public function emptyConfiguredFilterIsNotChangingRequestWithExistingFilter() { - $this->configuration->expects($this->exactly(2)) + $this->configuration->expects($this->any()) ->method('getIfExists') ->withConsecutive(['searching.size'], ['searching.facets']) ->will($this->onConsecutiveCalls(null, null)); - $this->configuration->expects($this->exactly(1)) + $this->configuration->expects($this->any()) ->method('get') - ->with('searching.filter') - ->willReturn(['anotherProperty' => '']); + ->will($this->onConsecutiveCalls( + ['anotherProperty' => ''], + $this->throwException(new InvalidArgumentException) + )); $this->connection->expects($this->once()) ->method('search') From 79aba3c544003afc6e2628f864aeb9edd1da5cf5 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 09:19:09 +0100 Subject: [PATCH 12/75] TASK: Add test cases for ResultItem Model Check whether all methods work as expected. E.g. we can retrieve data in all ways, but not change anything. --- Classes/Domain/Model/ResultItem.php | 2 +- Tests/Unit/Domain/Model/ResultItemTest.php | 113 +++++++++++++++++++++ 2 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 Tests/Unit/Domain/Model/ResultItemTest.php diff --git a/Classes/Domain/Model/ResultItem.php b/Classes/Domain/Model/ResultItem.php index f63b408..d0e370a 100644 --- a/Classes/Domain/Model/ResultItem.php +++ b/Classes/Domain/Model/ResultItem.php @@ -31,7 +31,7 @@ class ResultItem implements ResultItemInterface public function __construct(array $result) { - $this->data = $data; + $this->data = $result; } public function getPlainData() : array diff --git a/Tests/Unit/Domain/Model/ResultItemTest.php b/Tests/Unit/Domain/Model/ResultItemTest.php new file mode 100644 index 0000000..df11a5d --- /dev/null +++ b/Tests/Unit/Domain/Model/ResultItemTest.php @@ -0,0 +1,113 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Domain\Model\ResultItem; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class ResultItemTest extends AbstractUnitTestCase +{ + /** + * @test + */ + public function plainDataCanBeRetrieved() + { + $originalData = [ + 'uid' => 10, + 'title' => 'Some title', + ]; + $expectedData = $originalData; + + $subject = new ResultItem($originalData); + $this->assertSame( + $expectedData, + $subject->getPlainData(), + 'Could not retrieve plain data from result item.' + ); + } + + /** + * @test + */ + public function dataCanBeRetrievedInArrayNotation() + { + $originalData = [ + 'uid' => 10, + 'title' => 'Some title', + ]; + $expectedData = $originalData; + + $subject = new ResultItem($originalData); + $this->assertSame( + 'Some title', + $subject['title'], + 'Could not retrieve title in array notation.' + ); + } + + /** + * @test + */ + public function existenceOfDataCanBeChecked() + { + $originalData = [ + 'uid' => 10, + 'title' => 'Some title', + ]; + $expectedData = $originalData; + + $subject = new ResultItem($originalData); + $this->assertTrue(isset($subject['title']), 'Could not determine that title exists.'); + $this->assertFalse(isset($subject['title2']), 'Could not determine that title2 does not exists.'); + } + + /** + * @test + */ + public function dataCanNotBeChanged() + { + $originalData = [ + 'uid' => 10, + 'title' => 'Some title', + ]; + $expectedData = $originalData; + + $subject = new ResultItem($originalData); + $this->expectException(\BadMethodCallException::class); + $subject['title'] = 'New Title'; + } + + /** + * @test + */ + public function dataCanNotBeRemoved() + { + $originalData = [ + 'uid' => 10, + 'title' => 'Some title', + ]; + $expectedData = $originalData; + + $subject = new ResultItem($originalData); + $this->expectException(\BadMethodCallException::class); + unset($subject['title']); + } +} From cf91251be36e08cd749457adbbf1e1ce5404f15f Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 09:36:39 +0100 Subject: [PATCH 13/75] TASK :Add tests for SearchRequest Make sure exceptions with helpful messages are thrown if one object is missing when execute is called. Also make sure the expected methods are called. --- Classes/Domain/Model/SearchRequest.php | 18 +++-- Tests/Unit/Domain/Model/SearchRequestTest.php | 69 +++++++++++++++++-- 2 files changed, 75 insertions(+), 12 deletions(-) diff --git a/Classes/Domain/Model/SearchRequest.php b/Classes/Domain/Model/SearchRequest.php index d835e7b..69237bb 100644 --- a/Classes/Domain/Model/SearchRequest.php +++ b/Classes/Domain/Model/SearchRequest.php @@ -158,14 +158,20 @@ class SearchRequest implements SearchRequestInterface // Current implementation covers only paginate widget support. public function execute($returnRawQueryResult = false) { - if ($this->connection instanceof ConnectionInterface) { - return $this->searchService->processResult($this->connection->search($this)); + if (! ($this->connection instanceof ConnectionInterface)) { + throw new \InvalidArgumentException( + 'Connection was not set before, therefore execute can not work. Use `setConnection` before.', + 1502197732 + ); + } + if (! ($this->searchService instanceof SearchService)) { + throw new \InvalidArgumentException( + 'SearchService was not set before, therefore execute can not work. Use `setSearchService` before.', + 1520325175 + ); } - throw new \InvalidArgumentException( - 'Connection was not set before, therefore execute can not work. Use `setConnection` before.', - 1502197732 - ); + return $this->searchService->processResult($this->connection->search($this)); } public function setLimit($limit) diff --git a/Tests/Unit/Domain/Model/SearchRequestTest.php b/Tests/Unit/Domain/Model/SearchRequestTest.php index 127baf2..28f90e3 100644 --- a/Tests/Unit/Domain/Model/SearchRequestTest.php +++ b/Tests/Unit/Domain/Model/SearchRequestTest.php @@ -20,7 +20,10 @@ namespace Codappix\SearchCore\Tests\Unit\Domain\Model; * 02110-1301, USA. */ +use Codappix\SearchCore\Connection\ConnectionInterface; +use Codappix\SearchCore\Connection\SearchResultInterface; use Codappix\SearchCore\Domain\Model\SearchRequest; +use Codappix\SearchCore\Domain\Search\SearchService; use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; class SearchRequestTest extends AbstractUnitTestCase @@ -31,12 +34,12 @@ class SearchRequestTest extends AbstractUnitTestCase */ public function emptyFilterWillNotBeSet(array $filter) { - $searchRequest = new SearchRequest(); - $searchRequest->setFilter($filter); + $subject = new SearchRequest(); + $subject->setFilter($filter); $this->assertSame( [], - $searchRequest->getFilter(), + $subject->getFilter(), 'Empty filter were set, even if they should not.' ); } @@ -68,13 +71,67 @@ class SearchRequestTest extends AbstractUnitTestCase public function filterIsSet() { $filter = ['someField' => 'someValue']; - $searchRequest = new SearchRequest(); - $searchRequest->setFilter($filter); + $subject = new SearchRequest(); + $subject->setFilter($filter); $this->assertSame( $filter, - $searchRequest->getFilter(), + $subject->getFilter(), 'Filter was not set.' ); } + + /** + * @test + */ + public function exceptionIsThrownIfSearchServiceWasNotSet() + { + $subject = new SearchRequest(); + $subject->setConnection($this->getMockBuilder(ConnectionInterface::class)->getMock()); + $this->expectException(\InvalidArgumentException::class); + $subject->execute(); + } + + /** + * @test + */ + public function exceptionIsThrownIfConnectionWasNotSet() + { + $subject = new SearchRequest(); + $subject->setSearchService( + $this->getMockBuilder(SearchService::class) + ->disableOriginalConstructor() + ->getMock() + ); + $this->expectException(\InvalidArgumentException::class); + $subject->execute(); + } + + /** + * @test + */ + public function executionMakesUseOfProvidedConnectionAndSearchService() + { + $searchServiceMock = $this->getMockBuilder(SearchService::class) + ->disableOriginalConstructor() + ->getMock(); + $connectionMock = $this->getMockBuilder(ConnectionInterface::class) + ->getMock(); + $searchResultMock = $this->getMockBuilder(SearchResultInterface::class) + ->getMock(); + + $subject = new SearchRequest(); + $subject->setSearchService($searchServiceMock); + $subject->setConnection($connectionMock); + + $connectionMock->expects($this->once()) + ->method('search') + ->with($subject) + ->willReturn($searchResultMock); + $searchServiceMock->expects($this->once()) + ->method('processResult') + ->with($searchResultMock); + + $subject->execute(); + } } From 45bb12cf519bc51f69a15fff38e56fd1bdba86cd Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 09:44:26 +0100 Subject: [PATCH 14/75] TASK: Add tests for search result model --- Tests/Unit/Domain/Model/SearchResultTest.php | 100 +++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 Tests/Unit/Domain/Model/SearchResultTest.php diff --git a/Tests/Unit/Domain/Model/SearchResultTest.php b/Tests/Unit/Domain/Model/SearchResultTest.php new file mode 100644 index 0000000..910e06e --- /dev/null +++ b/Tests/Unit/Domain/Model/SearchResultTest.php @@ -0,0 +1,100 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Connection\ResultItemInterface; +use Codappix\SearchCore\Connection\SearchResultInterface; +use Codappix\SearchCore\Domain\Model\SearchResult; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class SearchResultTest extends AbstractUnitTestCase +{ + /** + * @test + */ + public function countIsRetrievedFromOriginalResult() + { + $originalSearchResultMock = $this->getMockBuilder(SearchResultInterface::class)->getMock(); + $originalSearchResultMock->expects($this->once())->method('count'); + + $subject = new SearchResult($originalSearchResultMock, []); + $subject->count(); + } + + /** + * @test + */ + public function currentCountIsRetrievedFromOriginalResult() + { + $originalSearchResultMock = $this->getMockBuilder(SearchResultInterface::class)->getMock(); + $originalSearchResultMock->expects($this->once())->method('getCurrentCount'); + + $subject = new SearchResult($originalSearchResultMock, []); + $subject->getCurrentCount(); + } + + /** + * @test + */ + public function facetsAreRetrievedFromOriginalResult() + { + $originalSearchResultMock = $this->getMockBuilder(SearchResultInterface::class)->getMock(); + $originalSearchResultMock->expects($this->once())->method('getFacets'); + + $subject = new SearchResult($originalSearchResultMock, []); + $subject->getFacets(); + } + + /** + * @test + */ + public function resultItemsCanBeRetrieved() + { + $originalSearchResultMock = $this->getMockBuilder(SearchResultInterface::class)->getMock(); + $data = [ + [ + 'uid' => 10, + 'title' => 'Some Title', + ], + [ + 'uid' => 11, + 'title' => 'Some Title 2', + ], + [ + 'uid' => 12, + 'title' => 'Some Title 3', + ], + ]; + + $subject = new SearchResult($originalSearchResultMock, $data); + $resultItems = $subject->getResults(); + + $this->assertCount(3, $resultItems); + + $this->assertSame(10, $resultItems[0]['uid']); + $this->assertSame(11, $resultItems[1]['uid']); + $this->assertSame(12, $resultItems[2]['uid']); + + $this->assertInstanceOf(ResultItemInterface::class, $resultItems[0]); + $this->assertInstanceOf(ResultItemInterface::class, $resultItems[1]); + $this->assertInstanceOf(ResultItemInterface::class, $resultItems[2]); + } +} From 0210110ccfcdee1e22316ddbfc94dd26c499bf86 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 10:26:39 +0100 Subject: [PATCH 15/75] TASK: Add test for applied data processing on search result --- .../Unit/Domain/Search/SearchServiceTest.php | 91 +++++++++++++++++-- 1 file changed, 81 insertions(+), 10 deletions(-) diff --git a/Tests/Unit/Domain/Search/SearchServiceTest.php b/Tests/Unit/Domain/Search/SearchServiceTest.php index aaf2e5c..6263f23 100644 --- a/Tests/Unit/Domain/Search/SearchServiceTest.php +++ b/Tests/Unit/Domain/Search/SearchServiceTest.php @@ -1,5 +1,5 @@ @@ -26,6 +26,7 @@ use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Connection\SearchResultInterface; use Codappix\SearchCore\DataProcessing\Service as DataProcessorService; use Codappix\SearchCore\Domain\Model\SearchRequest; +use Codappix\SearchCore\Domain\Model\SearchResult; use Codappix\SearchCore\Domain\Search\SearchService; use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; @@ -72,9 +73,6 @@ class SearchServiceTest extends AbstractUnitTestCase $this->connection = $this->getMockBuilder(ConnectionInterface::class) ->disableOriginalConstructor() ->getMock(); - $this->connection->expects($this->any()) - ->method('search') - ->willReturn($this->result); $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class) ->disableOriginalConstructor() ->getMock(); @@ -109,7 +107,8 @@ class SearchServiceTest extends AbstractUnitTestCase ->method('search') ->with($this->callback(function ($searchRequest) { return $searchRequest->getLimit() === 45; - })); + })) + ->willReturn($this->getMockBuilder(SearchResultInterface::class)->getMock()); $searchRequest = new SearchRequest('SearchWord'); $this->subject->search($searchRequest); @@ -131,7 +130,8 @@ class SearchServiceTest extends AbstractUnitTestCase ->method('search') ->with($this->callback(function ($searchRequest) { return $searchRequest->getLimit() === 10; - })); + })) + ->willReturn($this->getMockBuilder(SearchResultInterface::class)->getMock()); $searchRequest = new SearchRequest('SearchWord'); $this->subject->search($searchRequest); @@ -157,7 +157,8 @@ class SearchServiceTest extends AbstractUnitTestCase ->method('search') ->with($this->callback(function ($searchRequest) { return $searchRequest->getFilter() === ['property' => 'something']; - })); + })) + ->willReturn($this->getMockBuilder(SearchResultInterface::class)->getMock()); $searchRequest = new SearchRequest('SearchWord'); $this->subject->search($searchRequest); @@ -186,7 +187,8 @@ class SearchServiceTest extends AbstractUnitTestCase 'anotherProperty' => 'anything', 'property' => 'something', ]; - })); + })) + ->willReturn($this->getMockBuilder(SearchResultInterface::class)->getMock()); $searchRequest = new SearchRequest('SearchWord'); $searchRequest->setFilter(['anotherProperty' => 'anything']); @@ -210,7 +212,8 @@ class SearchServiceTest extends AbstractUnitTestCase ->method('search') ->with($this->callback(function ($searchRequest) { return $searchRequest->getFilter() === ['anotherProperty' => 'anything']; - })); + })) + ->willReturn($this->getMockBuilder(SearchResultInterface::class)->getMock()); $searchRequest = new SearchRequest('SearchWord'); $searchRequest->setFilter(['anotherProperty' => 'anything']); @@ -237,10 +240,78 @@ class SearchServiceTest extends AbstractUnitTestCase ->method('search') ->with($this->callback(function ($searchRequest) { return $searchRequest->getFilter() === ['anotherProperty' => 'anything']; - })); + })) + ->willReturn($this->getMockBuilder(SearchResultInterface::class)->getMock()); $searchRequest = new SearchRequest('SearchWord'); $searchRequest->setFilter(['anotherProperty' => 'anything']); $this->subject->search($searchRequest); } + + /** + * @test + */ + public function originalSearchResultIsReturnedIfNoDataProcessorIsConfigured() + { + $this->configuration->expects($this->any()) + ->method('getIfExists') + ->withConsecutive(['searching.size'], ['searching.facets']) + ->will($this->onConsecutiveCalls(null, null)); + $this->configuration->expects($this->any()) + ->method('get') + ->will($this->throwException(new InvalidArgumentException)); + + $searchResultMock = $this->getMockBuilder(SearchResultInterface::class)->getMock(); + + $this->connection->expects($this->once()) + ->method('search') + ->willReturn($searchResultMock); + + $this->dataProcessorService->expects($this->never())->method('executeDataProcessor'); + + $searchRequest = new SearchRequest(''); + $this->assertSame($searchResultMock, $this->subject->search($searchRequest), 'Did not get created result without applied data processing'); + } + + /** + * @test + */ + public function configuredDataProcessorsAreExecutedOnSearchResult() + { + $this->configuration->expects($this->any()) + ->method('getIfExists') + ->withConsecutive(['searching.size'], ['searching.facets']) + ->will($this->onConsecutiveCalls(null, null)); + $this->configuration->expects($this->any()) + ->method('get') + ->will($this->onConsecutiveCalls( + $this->throwException(new InvalidArgumentException), + ['SomeProcessorClass'] + )); + + $searchResultMock = $this->getMockBuilder(SearchResultInterface::class)->getMock(); + $searchResult = new SearchResult($searchResultMock, [['field 1' => 'value 1']]); + + $this->connection->expects($this->once()) + ->method('search') + ->willReturn($searchResult); + + $this->dataProcessorService->expects($this->once()) + ->method('executeDataProcessor') + ->with('SomeProcessorClass', ['field 1' => 'value 1']) + ->willReturn([ + 'field 1' => 'value 1', + 'field 2' => 'value 2', + ]); + + $this->objectManager->expects($this->once()) + ->method('get') + ->with(SearchResult::class, $searchResult, [ + ['field 1' => 'value 1', 'field 2' => 'value 2'] + ]) + ->willReturn($searchResultMock); + + $searchRequest = new SearchRequest(''); + $this->assertSame($searchResultMock, $this->subject->search($searchRequest), 'Did not get created result with applied data processing'); + } } From 176c466d7e49f072ed11ca23c03b5c84608b33ed Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 10:47:14 +0100 Subject: [PATCH 16/75] TASK: Update documentation for data processing --- Documentation/source/concepts.rst | 2 + .../dataProcessing/availableAndPlanned.rst | 32 ++++++++++++++++ .../source/configuration/indexing.rst | 37 ++----------------- .../source/configuration/searching.rst | 34 +++++++++++++++++ Documentation/source/features.rst | 10 +++++ 5 files changed, 81 insertions(+), 34 deletions(-) create mode 100644 Documentation/source/configuration/dataProcessing/availableAndPlanned.rst diff --git a/Documentation/source/concepts.rst b/Documentation/source/concepts.rst index 0fe6c5d..f4c5bbd 100644 --- a/Documentation/source/concepts.rst +++ b/Documentation/source/concepts.rst @@ -36,5 +36,7 @@ DataProcessing Before data is transfered to search service, it can be processed by "DataProcessors" like already known by :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing` of :ref:`t3tsref:cobj-fluidtemplate`. +The same is true for retrieved search results. They can be processed again by "DataProcessors" to +prepare data for display in Templates or further usage. Configuration is done through TypoScript, see :ref:`dataProcessing`. diff --git a/Documentation/source/configuration/dataProcessing/availableAndPlanned.rst b/Documentation/source/configuration/dataProcessing/availableAndPlanned.rst new file mode 100644 index 0000000..8284820 --- /dev/null +++ b/Documentation/source/configuration/dataProcessing/availableAndPlanned.rst @@ -0,0 +1,32 @@ +The following Processor are available: + +.. toctree:: + :maxdepth: 1 + :glob: + + /configuration/dataProcessing/CopyToProcessor + /configuration/dataProcessing/RemoveProcessor + /configuration/dataProcessing/GeoPointProcessor + +The following Processor are planned: + + ``Codappix\SearchCore\DataProcessing\ReplaceProcessor`` + Will execute a search and replace on configured fields. + + ``Codappix\SearchCore\DataProcessing\RootLevelProcessor`` + Will attach the root level to the record. + + ``Codappix\SearchCore\DataProcessing\ChannelProcessor`` + Will add a configurable channel to the record, e.g. if you have different areas in your + website like "products" and "infos". + + ``Codappix\SearchCore\DataProcessing\RelationResolverProcessor`` + Resolves all relations using the TCA. + +Of course you are able to provide further processors. Just implement +``Codappix\SearchCore\DataProcessing\ProcessorInterface`` and use the FQCN (=Fully qualified +class name) as done in the examples above. + +By implementing also the same interface as necessary for TYPO3 +:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`, you are able to reuse the same code +also for Fluid to prepare the same record fetched from DB for your fluid. diff --git a/Documentation/source/configuration/indexing.rst b/Documentation/source/configuration/indexing.rst index b605008..7d6c3d4 100644 --- a/Documentation/source/configuration/indexing.rst +++ b/Documentation/source/configuration/indexing.rst @@ -146,7 +146,7 @@ Example:: dataProcessing -------------- -Used by: All connections while indexing. +Used by: All connections while indexing, due to implementation inside ``AbstractIndexer``. Configure modifications on each document before sending it to the configured connection. Same as provided by TYPO3 for :ref:`t3tsref:cobj-fluidtemplate` through @@ -170,38 +170,7 @@ Example:: The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards all fields, including ``search_spellcheck`` will be copied to ``search_all``. -E.g. used to index all information into a field for :ref:`spellchecking` or searching with -different :ref:`mapping`. -The following Processor are available: +.. include:: /configuration/dataProcessing/availableAndPlanned.rst -.. toctree:: - :maxdepth: 1 - :glob: - - dataProcessing/CopyToProcessor - dataProcessing/RemoveProcessor - dataProcessing/GeoPointProcessor - -The following Processor are planned: - - ``Codappix\SearchCore\DataProcessing\ReplaceProcessor`` - Will execute a search and replace on configured fields. - - ``Codappix\SearchCore\DataProcessing\RootLevelProcessor`` - Will attach the root level to the record. - - ``Codappix\SearchCore\DataProcessing\ChannelProcessor`` - Will add a configurable channel to the record, e.g. if you have different areas in your - website like "products" and "infos". - - ``Codappix\SearchCore\DataProcessing\RelationResolverProcessor`` - Resolves all relations using the TCA. - -Of course you are able to provide further processors. Just implement -``Codappix\SearchCore\DataProcessing\ProcessorInterface`` and use the FQCN (=Fully qualified -class name) as done in the examples above. - -By implementing also the same interface as necessary for TYPO3 -:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`, you are able to reuse the same code -also for Fluid to prepare the same record fetched from DB for your fluid. +Also data processors are available for search results too, see :ref:`searching_dataProcessing`. diff --git a/Documentation/source/configuration/searching.rst b/Documentation/source/configuration/searching.rst index ae73fad..bfed51e 100644 --- a/Documentation/source/configuration/searching.rst +++ b/Documentation/source/configuration/searching.rst @@ -216,3 +216,37 @@ Example:: } Only ``filter`` is allowed as value. Will submit an empty query to switch to filter mode. + +.. _searching_dataProcessing: + +dataProcessing +-------------- + +Used by: All connections while indexing, due to implementation inside ``SearchService``. + +Configure modifications on each document before returning search result. Same as provided by TYPO3 +for :ref:`t3tsref:cobj-fluidtemplate` through +:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`. + +All processors are applied in configured order. Allowing to work with already processed data. + +Example:: + + plugin.tx_searchcore.settings.searching.dataProcessing { + 1 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 1 { + to = search_spellcheck + } + + 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 2 { + to = search_all + } + } + +The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards +all fields, including ``search_spellcheck`` will be copied to ``search_all``. + +.. include:: /configuration/dataProcessing/availableAndPlanned.rst + +Also data processors are available while indexing too, see :ref:`dataProcessing`. diff --git a/Documentation/source/features.rst b/Documentation/source/features.rst index 8baf453..1a4abd6 100644 --- a/Documentation/source/features.rst +++ b/Documentation/source/features.rst @@ -29,6 +29,16 @@ Also multiple filter are supported. Filtering results by fields for string conte Facets / aggregates are also possible. Therefore a mapping has to be defined in TypoScript for indexing, and the facets itself while searching. +.. _features_dataProcessing: + +DataProcessing +============== + +DataProcessing, as known from ``FLUIDTEMPLATE`` is available while indexing and for search results. +Each item can be processed by multiple processor to prepare data for indexing and output. + +See :ref:`concepts_indexing_dataprocessing` in :ref:`concepts` section. + .. _features_planned: Planned From 04aaad12fe611b994f0c9e7f704e0c73fcfc7079 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 11:16:00 +0100 Subject: [PATCH 17/75] FEATURE: Provide ContentObjectDataProcessorAdapterProcessor Allow integrator to execute any existing data processor for content objects. Resolves: #118 --- ...entObjectDataProcessorAdapterProcessor.php | 59 +++++++++++++++++++ ...entObjectDataProcessorAdapterProcessor.rst | 32 ++++++++++ .../dataProcessing/availableAndPlanned.rst | 3 +- ...bjectDataProcessorAdapterProcessorTest.php | 55 +++++++++++++++++ 4 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 Classes/DataProcessing/ContentObjectDataProcessorAdapterProcessor.php create mode 100644 Documentation/source/configuration/dataProcessing/ContentObjectDataProcessorAdapterProcessor.rst create mode 100644 Tests/Functional/DataProcessing/ContentObjectDataProcessorAdapterProcessorTest.php diff --git a/Classes/DataProcessing/ContentObjectDataProcessorAdapterProcessor.php b/Classes/DataProcessing/ContentObjectDataProcessorAdapterProcessor.php new file mode 100644 index 0000000..322486b --- /dev/null +++ b/Classes/DataProcessing/ContentObjectDataProcessorAdapterProcessor.php @@ -0,0 +1,59 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use TYPO3\CMS\Core\Utility\GeneralUtility; +use TYPO3\CMS\Extbase\Service\TypoScriptService; +use TYPO3\CMS\Frontend\ContentObject\ContentObjectRenderer; + +/** + * Executes an existing TYPO3 DataProcessor on the given data. + */ +class ContentObjectDataProcessorAdapterProcessor implements ProcessorInterface +{ + /** + * @var TypoScriptService + */ + protected $typoScriptService; + + public function __construct(TypoScriptService $typoScriptService) + { + $this->typoScriptService = $typoScriptService; + } + + public function processData(array $data, array $configuration) : array + { + $dataProcessor = GeneralUtility::makeInstance($configuration['_dataProcessor']); + $contentObjectRenderer = GeneralUtility::makeInstance(ContentObjectRenderer::class); + + $contentObjectRenderer->data = $data; + if (isset($configuration['_table'])) { + $contentObjectRenderer->start($data, $configuration['_table']); + } + + return $dataProcessor->process( + $contentObjectRenderer, + [], + $this->typoScriptService->convertPlainArrayToTypoScriptArray($configuration), + $data + ); + } +} diff --git a/Documentation/source/configuration/dataProcessing/ContentObjectDataProcessorAdapterProcessor.rst b/Documentation/source/configuration/dataProcessing/ContentObjectDataProcessorAdapterProcessor.rst new file mode 100644 index 0000000..7d8f0b2 --- /dev/null +++ b/Documentation/source/configuration/dataProcessing/ContentObjectDataProcessorAdapterProcessor.rst @@ -0,0 +1,32 @@ +``Codappix\SearchCore\DataProcessing\ContentObjectDataProcessorAdapterProcessor`` +================================================================================= + +Will execute an existing TYPO3 data processor. + +Possible Options: + +``_dataProcessor`` + Necessary, defined which data processor to apply. Provide the same as you would to call the + processor. +``_table`` + Defines the "current" table as used by some processors, e.g. + ``TYPO3\CMS\Frontend\DataProcessing\FilesProcessor``. + +All further options are passed to the configured data processor. Therefore they are documented at +each data processor. + +Example:: + + plugin.tx_searchcore.settings.searching.dataProcessing { + 1 = Codappix\SearchCore\DataProcessing\ContentObjectDataProcessorAdapterProcessor + 1 { + _table = pages + _dataProcessor = TYPO3\CMS\Frontend\DataProcessing\FilesProcessor + + references.fieldName = media + as = images + } + } + +The above example will create a new field ``images`` with resolved FAL relations from ``media`` +field. diff --git a/Documentation/source/configuration/dataProcessing/availableAndPlanned.rst b/Documentation/source/configuration/dataProcessing/availableAndPlanned.rst index 0d9385b..9f31736 100644 --- a/Documentation/source/configuration/dataProcessing/availableAndPlanned.rst +++ b/Documentation/source/configuration/dataProcessing/availableAndPlanned.rst @@ -4,9 +4,10 @@ The following Processor are available: :maxdepth: 1 :glob: + /configuration/dataProcessing/ContentObjectDataProcessorAdapterProcessor /configuration/dataProcessing/CopyToProcessor - /configuration/dataProcessing/RemoveProcessor /configuration/dataProcessing/GeoPointProcessor + /configuration/dataProcessing/RemoveProcessor The following Processor are planned: diff --git a/Tests/Functional/DataProcessing/ContentObjectDataProcessorAdapterProcessorTest.php b/Tests/Functional/DataProcessing/ContentObjectDataProcessorAdapterProcessorTest.php new file mode 100644 index 0000000..871b205 --- /dev/null +++ b/Tests/Functional/DataProcessing/ContentObjectDataProcessorAdapterProcessorTest.php @@ -0,0 +1,55 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\DataProcessing\ContentObjectDataProcessorAdapterProcessor; +use Codappix\SearchCore\Tests\Functional\AbstractFunctionalTestCase; +use TYPO3\CMS\Extbase\Service\TypoScriptService; +use TYPO3\CMS\Frontend\DataProcessing\SplitProcessor; + +class ContentObjectDataProcessorAdapterProcessorTest extends AbstractFunctionalTestCase +{ + /** + * @test + */ + public function contentObjectDataProcessorIsExecuted() + { + $record = ['content' => 'value1, value2']; + $configuration = [ + '_dataProcessor' => SplitProcessor::class, + 'delimiter' => ',', + 'fieldName' => 'content', + 'as' => 'new_content', + ]; + $expectedData = [ + 'content' => 'value1, value2', + 'new_content' => ['value1', 'value2'], + ]; + + $subject = new ContentObjectDataProcessorAdapterProcessor(new TypoScriptService); + $processedData = $subject->processData($record, $configuration); + $this->assertSame( + $expectedData, + $processedData, + 'The processor did not return the expected processed record.' + ); + } +} From 5d1e7c41bc6bfdddea1a2818c6df3e8b6e373a7a Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 11:36:05 +0100 Subject: [PATCH 18/75] !!!|FEATURE: Pass facet configuration to search service Do not limit integrator in possibilities to configure. Therefore previously configure facets for a field need to be adjusted to contain full configuration for elasticsearch. See changelog. Resolves: #120 --- Classes/Connection/Elasticsearch/Facet.php | 23 +++--- Classes/Connection/FacetRequestInterface.php | 10 +-- Classes/Domain/Model/FacetRequest.php | 24 ++---- Classes/Domain/Search/QueryFactory.php | 6 +- Classes/Domain/Search/SearchService.php | 7 +- Documentation/source/changelog.rst | 8 ++ .../20180406-120-facet-configuration.rst | 40 ++++++++++ Documentation/source/index.rst | 1 + .../Connection/Elasticsearch/FacetTest.php | 75 +++++++++++++++++++ .../Connection/Elasticsearch/FilterTest.php | 33 -------- Tests/Functional/Fixtures/BasicSetup.ts | 6 -- Tests/Functional/Fixtures/Searching/Facet.ts | 17 +++++ Tests/Unit/Domain/Search/QueryFactoryTest.php | 4 +- 13 files changed, 167 insertions(+), 87 deletions(-) create mode 100644 Documentation/source/changelog.rst create mode 100644 Documentation/source/changelog/20180406-120-facet-configuration.rst create mode 100644 Tests/Functional/Connection/Elasticsearch/FacetTest.php create mode 100644 Tests/Functional/Fixtures/Searching/Facet.ts diff --git a/Classes/Connection/Elasticsearch/Facet.php b/Classes/Connection/Elasticsearch/Facet.php index e24a659..1142d88 100644 --- a/Classes/Connection/Elasticsearch/Facet.php +++ b/Classes/Connection/Elasticsearch/Facet.php @@ -45,25 +45,26 @@ class Facet implements FacetInterface */ protected $options = []; - public function __construct($name, array $aggregation, ConfigurationContainerInterface $configuration) + public function __construct(string $name, array $aggregation, ConfigurationContainerInterface $configuration) { $this->name = $name; $this->buckets = $aggregation['buckets']; - $this->field = $configuration->getIfExists('searching.facets.' . $this->name . '.field') ?: ''; + + $config = $configuration->getIfExists('searching.facets.' . $this->name) ?: []; + foreach ($config as $configEntry) { + if (isset($configEntry['field'])) { + $this->field = $configEntry['field']; + break; + } + } } - /** - * @return string - */ - public function getName() + public function getName() : string { return $this->name; } - /** - * @return string - */ - public function getField() + public function getField() : string { return $this->field; } @@ -73,7 +74,7 @@ class Facet implements FacetInterface * * @return array */ - public function getOptions() + public function getOptions() : array { $this->initOptions(); diff --git a/Classes/Connection/FacetRequestInterface.php b/Classes/Connection/FacetRequestInterface.php index 9352f96..cdc16b1 100644 --- a/Classes/Connection/FacetRequestInterface.php +++ b/Classes/Connection/FacetRequestInterface.php @@ -28,15 +28,11 @@ interface FacetRequestInterface /** * The identifier of the facet, used as key in arrays and to get the facet * from search request, etc. - * - * @return string */ - public function getIdentifier(); + public function getIdentifier() : string; /** - * The field to use for facet building. - * - * @return string + * The config to use for facet building. */ - public function getField(); + public function getConfig() : array; } diff --git a/Classes/Domain/Model/FacetRequest.php b/Classes/Domain/Model/FacetRequest.php index b1dbc4c..0d82ad3 100644 --- a/Classes/Domain/Model/FacetRequest.php +++ b/Classes/Domain/Model/FacetRequest.php @@ -30,37 +30,27 @@ class FacetRequest implements FacetRequestInterface protected $identifier = ''; /** - * @var string + * @var array */ - protected $field = ''; + protected $config = []; /** - * TODO: Add validation / exception? * As the facets come from configuration this might be a good idea to help * integrators find issues. - * - * @param string $identifier - * @param string $field */ - public function __construct($identifier, $field) + public function __construct(string $identifier, array $config) { $this->identifier = $identifier; - $this->field = $field; + $this->config = $config; } - /** - * @return string - */ - public function getIdentifier() + public function getIdentifier() : string { return $this->identifier; } - /** - * @return string - */ - public function getField() + public function getConfig() : array { - return $this->field; + return $this->config; } } diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 980b763..8e122ab 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -239,11 +239,7 @@ class QueryFactory foreach ($searchRequest->getFacets() as $facet) { $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ 'aggs' => [ - $facet->getIdentifier() => [ - 'terms' => [ - 'field' => $facet->getField(), - ], - ], + $facet->getIdentifier() => $facet->getConfig(), ], ]); } diff --git a/Classes/Domain/Search/SearchService.php b/Classes/Domain/Search/SearchService.php index 120ab3b..40c99bd 100644 --- a/Classes/Domain/Search/SearchService.php +++ b/Classes/Domain/Search/SearchService.php @@ -103,15 +103,10 @@ class SearchService } foreach ($facetsConfig as $identifier => $facetConfig) { - if (!isset($facetConfig['field']) || trim($facetConfig['field']) === '') { - // TODO: Finish throw - throw new \Exception('message', 1499171142); - } - $searchRequest->addFacet($this->objectManager->get( FacetRequest::class, $identifier, - $facetConfig['field'] + $facetConfig )); } } diff --git a/Documentation/source/changelog.rst b/Documentation/source/changelog.rst new file mode 100644 index 0000000..0d7f020 --- /dev/null +++ b/Documentation/source/changelog.rst @@ -0,0 +1,8 @@ +Changelog +========= + +.. toctree:: + :maxdepth: 1 + :glob: + + changelog/* diff --git a/Documentation/source/changelog/20180406-120-facet-configuration.rst b/Documentation/source/changelog/20180406-120-facet-configuration.rst new file mode 100644 index 0000000..53021d9 --- /dev/null +++ b/Documentation/source/changelog/20180406-120-facet-configuration.rst @@ -0,0 +1,40 @@ +Breacking Change 120 "Pass facets configuration to elasticsearch" +================================================================= + +In order to allow arbitrary facet configuration, we do not process the facet configuration anymore. +Instead integrators are able to configure facets for search service "as is". We just pipe the +configuration through. + +Therefore the following, which worked before, does not work anymore: + +.. code-block:: typoscript + :linenos: + :emphasize-lines: 4 + + plugin.tx_searchcore.settings.search { + facets { + category { + field = categories + } + } + } + +Instead you have to provide the full configuration yourself: + +.. code-block:: typoscript + :linenos: + :emphasize-lines: 4,6 + + plugin.tx_searchcore.settings.search { + facets { + category { + terms { + field = categories + } + } + } + } + +You need to add line 4 and 6, the additional level ``terms`` for elasticsearch. + +See :issue:`120`. diff --git a/Documentation/source/index.rst b/Documentation/source/index.rst index 5427fc9..2edfae3 100644 --- a/Documentation/source/index.rst +++ b/Documentation/source/index.rst @@ -15,3 +15,4 @@ Table of Contents connections indexer development + changelog diff --git a/Tests/Functional/Connection/Elasticsearch/FacetTest.php b/Tests/Functional/Connection/Elasticsearch/FacetTest.php new file mode 100644 index 0000000..c89bc10 --- /dev/null +++ b/Tests/Functional/Connection/Elasticsearch/FacetTest.php @@ -0,0 +1,75 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Domain\Index\IndexerFactory; +use Codappix\SearchCore\Domain\Model\SearchRequest; +use Codappix\SearchCore\Domain\Search\SearchService; +use TYPO3\CMS\Extbase\Object\ObjectManager; + +class FacetTest extends AbstractFunctionalTestCase +{ + protected function getTypoScriptFilesForFrontendRootPage() + { + return array_merge(parent::getTypoScriptFilesForFrontendRootPage(), ['EXT:search_core/Tests/Functional/Fixtures/Searching/Facet.ts']); + } + + protected function getDataSets() + { + return array_merge( + parent::getDataSets(), + ['Tests/Functional/Fixtures/Searching/Filter.xml'] + ); + } + + /** + * @test + */ + public function itsPossibleToFetchFacetsForField() + { + \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) + ->get(IndexerFactory::class) + ->getIndexer('tt_content') + ->indexAllDocuments() + ; + + $searchService = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) + ->get(SearchService::class); + + $searchRequest = new SearchRequest(); + $result = $searchService->search($searchRequest); + + $this->assertSame(1, count($result->getFacets()), 'Did not receive the single defined facet.'); + + $facet = current($result->getFacets()); + $this->assertSame('contentTypes', $facet->getName(), 'Name of facet was not as expected.'); + $this->assertSame('CType', $facet->getField(), 'Field of facet was not expected.'); + + $options = $facet->getOptions(); + $this->assertSame(2, count($options), 'Did not receive the expected number of possible options for facet.'); + $option = $options['HTML']; + $this->assertSame('HTML', $option->getName(), 'Option did not have expected Name.'); + $this->assertSame(1, $option->getCount(), 'Option did not have expected count.'); + $option = $options['Header']; + $this->assertSame('Header', $option->getName(), 'Option did not have expected Name.'); + $this->assertSame(1, $option->getCount(), 'Option did not have expected count.'); + } +} diff --git a/Tests/Functional/Connection/Elasticsearch/FilterTest.php b/Tests/Functional/Connection/Elasticsearch/FilterTest.php index e364a4a..8bbe0f0 100644 --- a/Tests/Functional/Connection/Elasticsearch/FilterTest.php +++ b/Tests/Functional/Connection/Elasticsearch/FilterTest.php @@ -58,37 +58,4 @@ class FilterTest extends AbstractFunctionalTestCase $this->assertSame(5, $result->getResults()[0]['uid'], 'Did not get the expected result entry.'); $this->assertSame(1, count($result), 'Did not receive the single filtered element.'); } - - /** - * @test - */ - public function itsPossibleToFetchFacetsForField() - { - \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) - ->get(IndexerFactory::class) - ->getIndexer('tt_content') - ->indexAllDocuments() - ; - - $searchService = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) - ->get(SearchService::class); - - $searchRequest = new SearchRequest('Search Word'); - $result = $searchService->search($searchRequest); - - $this->assertSame(1, count($result->getFacets()), 'Did not receive the single defined facet.'); - - $facet = current($result->getFacets()); - $this->assertSame('contentTypes', $facet->getName(), 'Name of facet was not as expected.'); - $this->assertSame('CType', $facet->getField(), 'Field of facet was not expected.'); - - $options = $facet->getOptions(); - $this->assertSame(2, count($options), 'Did not receive the expected number of possible options for facet.'); - $option = $options['HTML']; - $this->assertSame('HTML', $option->getName(), 'Option did not have expected Name.'); - $this->assertSame(1, $option->getCount(), 'Option did not have expected count.'); - $option = $options['Header']; - $this->assertSame('Header', $option->getName(), 'Option did not have expected Name.'); - $this->assertSame(1, $option->getCount(), 'Option did not have expected count.'); - } } diff --git a/Tests/Functional/Fixtures/BasicSetup.ts b/Tests/Functional/Fixtures/BasicSetup.ts index b7b0c6a..4129362 100644 --- a/Tests/Functional/Fixtures/BasicSetup.ts +++ b/Tests/Functional/Fixtures/BasicSetup.ts @@ -37,12 +37,6 @@ plugin { } searching { - facets { - contentTypes { - field = CType - } - } - fields { query = _all } diff --git a/Tests/Functional/Fixtures/Searching/Facet.ts b/Tests/Functional/Fixtures/Searching/Facet.ts new file mode 100644 index 0000000..10eeae6 --- /dev/null +++ b/Tests/Functional/Fixtures/Searching/Facet.ts @@ -0,0 +1,17 @@ +plugin { + tx_searchcore { + settings { + searching { + facets { + contentTypes { + terms { + field = CType + } + } + } + } + } + } +} + +module.tx_searchcore < plugin.tx_searchcore diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index e0fc86d..881cd8b 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -118,8 +118,8 @@ class QueryFactoryTest extends AbstractUnitTestCase { $this->configureConfigurationMockWithDefault(); $searchRequest = new SearchRequest('SearchWord'); - $searchRequest->addFacet(new FacetRequest('Identifier', 'FieldName')); - $searchRequest->addFacet(new FacetRequest('Identifier 2', 'FieldName 2')); + $searchRequest->addFacet(new FacetRequest('Identifier', ['terms' => ['field' => 'FieldName']])); + $searchRequest->addFacet(new FacetRequest('Identifier 2', ['terms' => ['field' => 'FieldName 2']])); $query = $this->subject->create($searchRequest); $this->assertSame( From a893303939178090c03f60ca6aed61673962e7e8 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 13:04:42 +0100 Subject: [PATCH 19/75] TASK: Improve ResultItemTest Do not repeat content, use variable which is also better to read. And do not add unnecessary, unused, variables. --- Tests/Unit/Domain/Model/ResultItemTest.php | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Tests/Unit/Domain/Model/ResultItemTest.php b/Tests/Unit/Domain/Model/ResultItemTest.php index df11a5d..53477c7 100644 --- a/Tests/Unit/Domain/Model/ResultItemTest.php +++ b/Tests/Unit/Domain/Model/ResultItemTest.php @@ -57,7 +57,7 @@ class ResultItemTest extends AbstractUnitTestCase $subject = new ResultItem($originalData); $this->assertSame( - 'Some title', + $originalData['title'], $subject['title'], 'Could not retrieve title in array notation.' ); @@ -72,7 +72,6 @@ class ResultItemTest extends AbstractUnitTestCase 'uid' => 10, 'title' => 'Some title', ]; - $expectedData = $originalData; $subject = new ResultItem($originalData); $this->assertTrue(isset($subject['title']), 'Could not determine that title exists.'); @@ -88,7 +87,6 @@ class ResultItemTest extends AbstractUnitTestCase 'uid' => 10, 'title' => 'Some title', ]; - $expectedData = $originalData; $subject = new ResultItem($originalData); $this->expectException(\BadMethodCallException::class); @@ -104,7 +102,6 @@ class ResultItemTest extends AbstractUnitTestCase 'uid' => 10, 'title' => 'Some title', ]; - $expectedData = $originalData; $subject = new ResultItem($originalData); $this->expectException(\BadMethodCallException::class); From 6456f315030fd6a60f007816ac0155328d92db4a Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 13:23:08 +0100 Subject: [PATCH 20/75] TASK: Make test more readable Make sure everyone knows what we compare, do not add hardcoded information. --- Tests/Unit/Domain/Model/SearchResultTest.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tests/Unit/Domain/Model/SearchResultTest.php b/Tests/Unit/Domain/Model/SearchResultTest.php index 910e06e..eebb3c1 100644 --- a/Tests/Unit/Domain/Model/SearchResultTest.php +++ b/Tests/Unit/Domain/Model/SearchResultTest.php @@ -89,9 +89,9 @@ class SearchResultTest extends AbstractUnitTestCase $this->assertCount(3, $resultItems); - $this->assertSame(10, $resultItems[0]['uid']); - $this->assertSame(11, $resultItems[1]['uid']); - $this->assertSame(12, $resultItems[2]['uid']); + $this->assertSame($data[0]['uid'], $resultItems[0]['uid']); + $this->assertSame($data[1]['uid'], $resultItems[1]['uid']); + $this->assertSame($data[2]['uid'], $resultItems[2]['uid']); $this->assertInstanceOf(ResultItemInterface::class, $resultItems[0]); $this->assertInstanceOf(ResultItemInterface::class, $resultItems[1]); From 3731bcf4743821a6906078a9f546848294f56b00 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 16:39:07 +0100 Subject: [PATCH 21/75] TASK: Fix CGL --- Classes/Domain/Search/SearchService.php | 3 +-- Tests/Unit/Domain/Search/SearchServiceTest.php | 12 ++++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Classes/Domain/Search/SearchService.php b/Classes/Domain/Search/SearchService.php index a331ab4..03fc11b 100644 --- a/Classes/Domain/Search/SearchService.php +++ b/Classes/Domain/Search/SearchService.php @@ -168,8 +168,7 @@ class SearchService $searchResult, $newSearchResultItems ); - } - catch (InvalidArgumentException $e) { + } catch (InvalidArgumentException $e) { return $searchResult; } } diff --git a/Tests/Unit/Domain/Search/SearchServiceTest.php b/Tests/Unit/Domain/Search/SearchServiceTest.php index 6263f23..3d6791e 100644 --- a/Tests/Unit/Domain/Search/SearchServiceTest.php +++ b/Tests/Unit/Domain/Search/SearchServiceTest.php @@ -270,7 +270,11 @@ class SearchServiceTest extends AbstractUnitTestCase $this->dataProcessorService->expects($this->never())->method('executeDataProcessor'); $searchRequest = new SearchRequest(''); - $this->assertSame($searchResultMock, $this->subject->search($searchRequest), 'Did not get created result without applied data processing'); + $this->assertSame( + $searchResultMock, + $this->subject->search($searchRequest), + 'Did not get created result without applied data processing' + ); } /** @@ -312,6 +316,10 @@ class SearchServiceTest extends AbstractUnitTestCase ->willReturn($searchResultMock); $searchRequest = new SearchRequest(''); - $this->assertSame($searchResultMock, $this->subject->search($searchRequest), 'Did not get created result with applied data processing'); + $this->assertSame( + $searchResultMock, + $this->subject->search($searchRequest), + 'Did not get created result with applied data processing' + ); } } From 6544ec07d315a6d3b1624d9a5add948d078e14cb Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 15:56:14 +0100 Subject: [PATCH 22/75] FEATURE: Support display name for facet option As some search services, like elasticsearch, allow generation of a string that should be displayed in frontend, we provide a new getter for that. The old existing name can be a fallback in custom implementations. --- .../Connection/Elasticsearch/FacetOption.php | 21 +++--- Classes/Connection/FacetOptionInterface.php | 14 ++-- .../Elasticsearch/FacetOptionTest.php | 64 +++++++++++++++++++ 3 files changed, 85 insertions(+), 14 deletions(-) create mode 100644 Tests/Unit/Connection/Elasticsearch/FacetOptionTest.php diff --git a/Classes/Connection/Elasticsearch/FacetOption.php b/Classes/Connection/Elasticsearch/FacetOption.php index 7359434..6b544bb 100644 --- a/Classes/Connection/Elasticsearch/FacetOption.php +++ b/Classes/Connection/Elasticsearch/FacetOption.php @@ -29,6 +29,11 @@ class FacetOption implements FacetOptionInterface */ protected $name = ''; + /** + * @var string + */ + protected $displayName = ''; + /** * @var int */ @@ -40,21 +45,21 @@ class FacetOption implements FacetOptionInterface public function __construct(array $bucket) { $this->name = $bucket['key']; + $this->displayName = isset($bucket['key_as_string']) ? $bucket['key_as_string'] : $this->getName(); $this->count = $bucket['doc_count']; } - /** - * @return string - */ - public function getName() + public function getName() : string { return $this->name; } - /** - * @return int - */ - public function getCount() + public function getDisplayName() : string + { + return $this->displayName; + } + + public function getCount() : int { return $this->count; } diff --git a/Classes/Connection/FacetOptionInterface.php b/Classes/Connection/FacetOptionInterface.php index 51a1efd..bf998db 100644 --- a/Classes/Connection/FacetOptionInterface.php +++ b/Classes/Connection/FacetOptionInterface.php @@ -28,15 +28,17 @@ interface FacetOptionInterface /** * Returns the name of this option. Equivalent * to value used for filtering. - * - * @return string */ - public function getName(); + public function getName() : string; + + /** + * If a pre-rendered name is provided, this will be returned. + * Otherwise it's the same as getName(). + */ + public function getDisplayName() : string; /** * Returns the number of found results for this option. - * - * @return int */ - public function getCount(); + public function getCount() : int; } diff --git a/Tests/Unit/Connection/Elasticsearch/FacetOptionTest.php b/Tests/Unit/Connection/Elasticsearch/FacetOptionTest.php new file mode 100644 index 0000000..696762c --- /dev/null +++ b/Tests/Unit/Connection/Elasticsearch/FacetOptionTest.php @@ -0,0 +1,64 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Connection\Elasticsearch\FacetOption; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class FacetOptionTest extends AbstractUnitTestCase +{ + /** + * @test + */ + public function displayNameIsReturnedAsExpected() + { + $bucket = [ + 'key' => 'Name', + 'key_as_string' => 'DisplayName', + 'doc_count' => 10, + ]; + $subject = new FacetOption($bucket); + + $this->assertSame( + $bucket['key_as_string'], + $subject->getDisplayName(), + 'Display name was not returned as expected.' + ); + } + + /** + * @test + */ + public function displayNameIsReturnedAsExpectedIfNotProvided() + { + $bucket = [ + 'key' => 'Name', + 'doc_count' => 10, + ]; + $subject = new FacetOption($bucket); + + $this->assertSame( + $bucket['key'], + $subject->getDisplayName(), + 'Display name was not returned as expected.' + ); + } +} From 88f301f22888f0b3ce411dcdc1b2af7252005296 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 16:34:59 +0100 Subject: [PATCH 23/75] FEATURE: Allow range queries for elasticsearch Allow "raw" configuration and support "range" type. Also prevent adding boosts if no search term was submitted which can be boosted. Resolves: #119 --- Classes/Domain/Search/QueryFactory.php | 16 ++++++ .../source/configuration/searching.rst | 47 ++++++++++++++--- Tests/Unit/Domain/Search/QueryFactoryTest.php | 52 +++++++++++++++++++ 3 files changed, 107 insertions(+), 8 deletions(-) diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 978d88a..dcf6ce5 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -121,6 +121,10 @@ class QueryFactory return; } + if (trim($searchRequest->getSearchTerm()) === '') { + return; + } + $boostQueryParts = []; foreach ($fields as $fieldName => $boostValue) { @@ -238,6 +242,18 @@ class QueryFactory } } + if (isset($config['raw'])) { + $filter = array_merge($config['raw'], $filter); + } + + if ($config['type'] === 'range') { + return [ + 'range' => [ + $config['field'] => $filter, + ], + ]; + } + return [$config['field'] => $filter]; } diff --git a/Documentation/source/configuration/searching.rst b/Documentation/source/configuration/searching.rst index bfed51e..857aa74 100644 --- a/Documentation/source/configuration/searching.rst +++ b/Documentation/source/configuration/searching.rst @@ -119,20 +119,48 @@ E.g. you submit a filter in form of: .. code-block:: html - - - + + Due to TYPO3 7.x fluid limitations, we build this input ourself. + No longer necessary in 8 and above + + + This will create a ``distance`` filter with subproperties. To make this filter actually work, you can add the following TypoScript, which will be added to the filter:: mapping { filter { - distance { - field = geo_distance + month { + type = range + field = released + raw { + format = yyyy-MM + } + fields { - distance = distance - location = location + gte = from + lte = to } } } @@ -143,9 +171,12 @@ in elasticsearch. In above example they do match, but you can also use different On the left hand side is the elasticsearch field name, on the right side the one submitted as a filter. -The ``field``, in above example ``geo_distance``, will be used as the elasticsearch field for +The ``field``, in above example ``released``, will be used as the elasticsearch field for filtering. This way you can use arbitrary filter names and map them to existing elasticsearch fields. +Everything that is configured inside ``raw`` is passed, as is, to search service, e.g. +elasticsearch. + .. _fields: fields diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index 881cd8b..dc824c5 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -86,6 +86,58 @@ class QueryFactoryTest extends AbstractUnitTestCase ); } + /** + * @test + */ + public function rangeFilterIsAddedToQuery() + { + $this->configureConfigurationMockWithDefault(); + $this->configuration->expects($this->any()) + ->method('getIfExists') + ->will($this->returnCallback(function ($configName) { + if ($configName === 'searching.mapping.filter.month') { + return [ + 'type' => 'range', + 'field' => 'released', + 'raw' => [ + 'format' => 'yyyy-MM', + ], + 'fields' => [ + 'gte' => 'from', + 'lte' => 'to', + ], + ]; + } + + return []; + })); + + $searchRequest = new SearchRequest('SearchWord'); + $searchRequest->setFilter([ + 'month' => [ + 'from' => '2016-03', + 'to' => '2017-11', + ], + ]); + + $query = $this->subject->create($searchRequest); + $this->assertSame( + [ + [ + 'range' => [ + 'released' => [ + 'format' => 'yyyy-MM', + 'gte' => '2016-03', + 'lte' => '2017-11', + ], + ], + ] + ], + $query->toArray()['query']['bool']['filter'], + 'Filter was not added to query.' + ); + } + /** * @test */ From 560597dcff89b4e7f9f5618af0005f1074f07c83 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 17:40:49 +0100 Subject: [PATCH 24/75] TASK: Cleanup type annotations and phpstan issues --- Classes/Command/IndexCommandController.php | 4 +-- .../Configuration/ConfigurationContainer.php | 5 ++- .../ConfigurationContainerInterface.php | 4 +-- Classes/Connection/ConnectionInterface.php | 30 ++++------------ Classes/Connection/Elasticsearch.php | 29 +++++---------- .../Connection/Elasticsearch/Connection.php | 10 +++--- .../Elasticsearch/DocumentFactory.php | 14 ++------ Classes/Connection/Elasticsearch/Facet.php | 1 + .../Connection/Elasticsearch/FacetOption.php | 10 ++---- .../Connection/Elasticsearch/IndexFactory.php | 21 ++--------- .../Elasticsearch/MappingFactory.php | 12 ++----- .../Connection/Elasticsearch/SearchResult.php | 6 ++-- .../Connection/Elasticsearch/TypeFactory.php | 7 +--- Classes/Connection/FacetInterface.php | 7 ++-- Classes/Connection/FacetOptionInterface.php | 8 ++--- Classes/Connection/SearchRequestInterface.php | 23 ++++++++---- Classes/Connection/SearchResultInterface.php | 8 ++--- Classes/Domain/Index/AbstractIndexer.php | 34 +++++------------- Classes/Domain/Index/IndexerFactory.php | 11 ++---- Classes/Domain/Index/IndexerInterface.php | 8 ++--- Classes/Domain/Index/TcaIndexer.php | 23 +++++------- .../Domain/Index/TcaIndexer/PagesIndexer.php | 15 ++------ .../Index/TcaIndexer/TcaTableService.php | 31 +++------------- Classes/Domain/Model/SearchRequest.php | 35 ++++-------------- Classes/Domain/Model/SearchResult.php | 6 ++-- Classes/Domain/Search/SearchService.php | 12 +------ Classes/Domain/Service/DataHandler.php | 23 +++--------- Classes/Hook/DataHandler.php | 36 ++++++------------- .../Form/Finisher/DataHandlerFinisher.php | 2 +- Classes/Utility/FrontendUtility.php | 5 +-- 30 files changed, 117 insertions(+), 323 deletions(-) diff --git a/Classes/Command/IndexCommandController.php b/Classes/Command/IndexCommandController.php index 70816ed..61aad53 100644 --- a/Classes/Command/IndexCommandController.php +++ b/Classes/Command/IndexCommandController.php @@ -48,7 +48,7 @@ class IndexCommandController extends CommandController * * @param string $identifier */ - public function indexCommand($identifier) + public function indexCommand(string $identifier) { try { $this->indexerFactory->getIndexer($identifier)->indexAllDocuments(); @@ -63,7 +63,7 @@ class IndexCommandController extends CommandController * * @param string $identifier */ - public function deleteCommand($identifier) + public function deleteCommand(string $identifier) { try { $this->indexerFactory->getIndexer($identifier)->delete(); diff --git a/Classes/Configuration/ConfigurationContainer.php b/Classes/Configuration/ConfigurationContainer.php index 7481692..eab76e0 100644 --- a/Classes/Configuration/ConfigurationContainer.php +++ b/Classes/Configuration/ConfigurationContainer.php @@ -39,7 +39,6 @@ class ConfigurationContainer implements ConfigurationContainerInterface /** * Inject settings via ConfigurationManager. * - * @param ConfigurationManagerInterface $configurationManager * @throws NoConfigurationException */ public function injectConfigurationManager(ConfigurationManagerInterface $configurationManager) @@ -59,7 +58,7 @@ class ConfigurationContainer implements ConfigurationContainerInterface * @return mixed * @throws InvalidArgumentException */ - public function get($path) + public function get(string $path) { $value = ArrayUtility::getValueByPath($this->settings, $path); @@ -77,7 +76,7 @@ class ConfigurationContainer implements ConfigurationContainerInterface * @param string $path In dot notation. * @return mixed */ - public function getIfExists($path) + public function getIfExists(string $path) { return ArrayUtility::getValueByPath($this->settings, $path); } diff --git a/Classes/Configuration/ConfigurationContainerInterface.php b/Classes/Configuration/ConfigurationContainerInterface.php index 9429535..1978453 100644 --- a/Classes/Configuration/ConfigurationContainerInterface.php +++ b/Classes/Configuration/ConfigurationContainerInterface.php @@ -37,7 +37,7 @@ interface ConfigurationContainerInterface extends Singleton * * @throws InvalidArgumentException */ - public function get($path); + public function get(string $path); /** * Same as get but will not throw an exception but return null. @@ -45,5 +45,5 @@ interface ConfigurationContainerInterface extends Singleton * @param string $path In dot notation. * @return mixed|null */ - public function getIfExists($path); + public function getIfExists(string $path); } diff --git a/Classes/Connection/ConnectionInterface.php b/Classes/Connection/ConnectionInterface.php index 59cf9f8..a336793 100644 --- a/Classes/Connection/ConnectionInterface.php +++ b/Classes/Connection/ConnectionInterface.php @@ -28,62 +28,44 @@ interface ConnectionInterface /** * Will add a new document. * - * @param string $documentType - * @param array $document - * * @return void */ - public function addDocument($documentType, array $document); + public function addDocument(string $documentType, array $document); /** * Add the given documents. * - * @param string $documentType - * @param array $documents - * * @return void */ - public function addDocuments($documentType, array $documents); + public function addDocuments(string $documentType, array $documents); /** * Will update an existing document. * * NOTE: Batch updating is not yet supported. * - * @param string $documentType - * @param array $document - * * @return void */ - public function updateDocument($documentType, array $document); + public function updateDocument(string $documentType, array $document); /** * Will remove an existing document. * * NOTE: Batch deleting is not yet supported. * - * @param string $documentType - * @param int $identifier - * * @return void */ - public function deleteDocument($documentType, $identifier); + public function deleteDocument(string $documentType, string $identifier); /** * Search by given request and return result. - * - * @param SearchRequestInterface $searchRequest - * - * @return SearchResultInterface */ - public function search(SearchRequestInterface $searchRequest); + public function search(SearchRequestInterface $searchRequest) : SearchResultInterface; /** * Will delete the whole index / db. * - * @param string $documentType - * * @return void */ - public function deleteIndex($documentType); + public function deleteIndex(string $documentType); } diff --git a/Classes/Connection/Elasticsearch.php b/Classes/Connection/Elasticsearch.php index b86bebf..555d4e7 100644 --- a/Classes/Connection/Elasticsearch.php +++ b/Classes/Connection/Elasticsearch.php @@ -112,7 +112,7 @@ class Elasticsearch implements Singleton, ConnectionInterface $this->queryFactory = $queryFactory; } - public function addDocument($documentType, array $document) + public function addDocument(string $documentType, array $document) { $this->withType( $documentType, @@ -122,7 +122,7 @@ class Elasticsearch implements Singleton, ConnectionInterface ); } - public function deleteDocument($documentType, $identifier) + public function deleteDocument(string $documentType, string $identifier) { try { $this->withType( @@ -139,7 +139,7 @@ class Elasticsearch implements Singleton, ConnectionInterface } } - public function updateDocument($documentType, array $document) + public function updateDocument(string $documentType, array $document) { $this->withType( $documentType, @@ -149,7 +149,7 @@ class Elasticsearch implements Singleton, ConnectionInterface ); } - public function addDocuments($documentType, array $documents) + public function addDocuments(string $documentType, array $documents) { $this->withType( $documentType, @@ -159,7 +159,7 @@ class Elasticsearch implements Singleton, ConnectionInterface ); } - public function deleteIndex($documentType) + public function deleteIndex(string $documentType) { $index = $this->connection->getClient()->getIndex('typo3content'); @@ -173,11 +173,8 @@ class Elasticsearch implements Singleton, ConnectionInterface /** * Execute given callback with Elastica Type based on provided documentType - * - * @param string $documentType - * @param callable $callback */ - protected function withType($documentType, callable $callback) + protected function withType(string $documentType, callable $callback) { $type = $this->getType($documentType); // TODO: Check whether it's to heavy to send it so often e.g. for every single document. @@ -191,12 +188,7 @@ class Elasticsearch implements Singleton, ConnectionInterface $type->getIndex()->refresh(); } - /** - * @param SearchRequestInterface $searchRequest - * - * @return SearchResultInterface - */ - public function search(SearchRequestInterface $searchRequest) + public function search(SearchRequestInterface $searchRequest) : SearchResultInterface { $this->logger->debug('Search for', [$searchRequest->getSearchTerm()]); @@ -207,12 +199,7 @@ class Elasticsearch implements Singleton, ConnectionInterface return $this->objectManager->get(SearchResult::class, $searchRequest, $search->search()); } - /** - * @param string $documentType - * - * @return \Elastica\Type - */ - protected function getType($documentType) + protected function getType(string $documentType) : \Elastica\Type { return $this->typeFactory->getType( $this->indexFactory->getIndex( diff --git a/Classes/Connection/Elasticsearch/Connection.php b/Classes/Connection/Elasticsearch/Connection.php index cd0a1fd..10a62d3 100644 --- a/Classes/Connection/Elasticsearch/Connection.php +++ b/Classes/Connection/Elasticsearch/Connection.php @@ -52,9 +52,8 @@ class Connection implements Singleton ) { $this->configuration = $configuration; - $this->elasticaClient = $elasticaClient; - if ($this->elasticaClient === null) { - $this->elasticaClient = new \Elastica\Client([ + if ($elasticaClient === null) { + $elasticaClient = new \Elastica\Client([ 'host' => $this->configuration->get('connections.elasticsearch.host'), 'port' => $this->configuration->get('connections.elasticsearch.port'), // TODO: Make configurable @@ -63,14 +62,13 @@ class Connection implements Singleton // TODO: Make configurable. // new \Elastica\Log($this->elasticaClient); } + $this->elasticaClient = $elasticaClient; } /** * Get the concrete client for internal usage! - * - * @return \Elastica\Client */ - public function getClient() + public function getClient() : \ Elastica\Client { return $this->elasticaClient; } diff --git a/Classes/Connection/Elasticsearch/DocumentFactory.php b/Classes/Connection/Elasticsearch/DocumentFactory.php index 390c592..beb091a 100644 --- a/Classes/Connection/Elasticsearch/DocumentFactory.php +++ b/Classes/Connection/Elasticsearch/DocumentFactory.php @@ -44,13 +44,8 @@ class DocumentFactory implements Singleton /** * Creates document from document. - * - * @param string $documentType - * @param array $document - * - * @return \Elastica\Document */ - public function getDocument($documentType, array $document) + public function getDocument(string $documentType, array $document) : \Elastica\Document { // TODO: Use DocumentType for further configuration. @@ -70,13 +65,8 @@ class DocumentFactory implements Singleton /** * Creates documents based on documents. - * - * @param string $documentType - * @param array $documents - * - * @return array */ - public function getDocuments($documentType, array $documents) + public function getDocuments(string $documentType, array $documents) : array { foreach ($documents as &$document) { $document = $this->getDocument($documentType, $document); diff --git a/Classes/Connection/Elasticsearch/Facet.php b/Classes/Connection/Elasticsearch/Facet.php index 1142d88..27de076 100644 --- a/Classes/Connection/Elasticsearch/Facet.php +++ b/Classes/Connection/Elasticsearch/Facet.php @@ -22,6 +22,7 @@ namespace Codappix\SearchCore\Connection\Elasticsearch; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Connection\FacetInterface; +use Codappix\SearchCore\Connection\FacetOptionInterface; class Facet implements FacetInterface { diff --git a/Classes/Connection/Elasticsearch/FacetOption.php b/Classes/Connection/Elasticsearch/FacetOption.php index 7359434..300a505 100644 --- a/Classes/Connection/Elasticsearch/FacetOption.php +++ b/Classes/Connection/Elasticsearch/FacetOption.php @@ -43,18 +43,12 @@ class FacetOption implements FacetOptionInterface $this->count = $bucket['doc_count']; } - /** - * @return string - */ - public function getName() + public function getName() : string { return $this->name; } - /** - * @return int - */ - public function getCount() + public function getCount() : int { return $this->count; } diff --git a/Classes/Connection/Elasticsearch/IndexFactory.php b/Classes/Connection/Elasticsearch/IndexFactory.php index 66b448b..6ddab67 100644 --- a/Classes/Connection/Elasticsearch/IndexFactory.php +++ b/Classes/Connection/Elasticsearch/IndexFactory.php @@ -49,13 +49,8 @@ class IndexFactory implements Singleton /** * Get an index bases on TYPO3 table name. - * - * @param Connection $connection - * @param string $documentType - * - * @return \Elastica\Index */ - public function getIndex(Connection $connection, $documentType) + public function getIndex(Connection $connection, string $documentType) : \Elastica\Index { $index = $connection->getClient()->getIndex('typo3content'); @@ -66,12 +61,7 @@ class IndexFactory implements Singleton return $index; } - /** - * @param string $documentType - * - * @return array - */ - protected function getConfigurationFor($documentType) + protected function getConfigurationFor(string $documentType) : array { try { $configuration = $this->configuration->get('indexing.' . $documentType . '.index'); @@ -88,12 +78,7 @@ class IndexFactory implements Singleton } } - /** - * @param array $analyzer - * - * @return array - */ - protected function prepareAnalyzerConfiguration(array $analyzer) + protected function prepareAnalyzerConfiguration(array $analyzer) : array { $fieldsToExplode = ['char_filter', 'filter']; diff --git a/Classes/Connection/Elasticsearch/MappingFactory.php b/Classes/Connection/Elasticsearch/MappingFactory.php index 44a05c1..3882556 100644 --- a/Classes/Connection/Elasticsearch/MappingFactory.php +++ b/Classes/Connection/Elasticsearch/MappingFactory.php @@ -44,12 +44,8 @@ class MappingFactory implements Singleton /** * Get an mapping based on type. - * - * @param \Elastica\Type $type - * - * @return \Elastica\Mapping */ - public function getMapping(\Elastica\Type $type) + public function getMapping(\Elastica\Type $type) : \Elastica\Type\Mapping { $mapping = new \Elastica\Type\Mapping(); $mapping->setType($type); @@ -64,11 +60,7 @@ class MappingFactory implements Singleton return $mapping; } - /** - * @param string $identifier - * @return array - */ - protected function getConfiguration($identifier) + protected function getConfiguration(string $identifier) : array { try { return $this->configuration->get('indexing.' . $identifier . '.mapping'); diff --git a/Classes/Connection/Elasticsearch/SearchResult.php b/Classes/Connection/Elasticsearch/SearchResult.php index 867d823..5b3d381 100644 --- a/Classes/Connection/Elasticsearch/SearchResult.php +++ b/Classes/Connection/Elasticsearch/SearchResult.php @@ -77,7 +77,7 @@ class SearchResult implements SearchResultInterface /** * @return array */ - public function getResults() + public function getResults() : array { $this->initResults(); @@ -89,14 +89,14 @@ class SearchResult implements SearchResultInterface * * @return array */ - public function getFacets() + public function getFacets() : array { $this->initFacets(); return $this->facets; } - public function getCurrentCount() + public function getCurrentCount() : int { return $this->result->count(); } diff --git a/Classes/Connection/Elasticsearch/TypeFactory.php b/Classes/Connection/Elasticsearch/TypeFactory.php index d5283f8..de8e336 100644 --- a/Classes/Connection/Elasticsearch/TypeFactory.php +++ b/Classes/Connection/Elasticsearch/TypeFactory.php @@ -32,13 +32,8 @@ class TypeFactory implements Singleton { /** * Get an index bases on TYPO3 table name. - * - * @param \Elastica\Index $index - * @param string $documentType - * - * @return \Elastica\Type */ - public function getType(\Elastica\Index $index, $documentType) + public function getType(\Elastica\Index $index, string $documentType) : \Elastica\Type { return $index->getType($documentType); } diff --git a/Classes/Connection/FacetInterface.php b/Classes/Connection/FacetInterface.php index b1cc421..3ec549d 100644 --- a/Classes/Connection/FacetInterface.php +++ b/Classes/Connection/FacetInterface.php @@ -25,15 +25,12 @@ namespace Codappix\SearchCore\Connection; */ interface FacetInterface { - /** - * @return string - */ - public function getName(); + public function getName() : string; /** * Returns all possible options for this facet. * * @return array */ - public function getOptions(); + public function getOptions() : array; } diff --git a/Classes/Connection/FacetOptionInterface.php b/Classes/Connection/FacetOptionInterface.php index 51a1efd..cb1c529 100644 --- a/Classes/Connection/FacetOptionInterface.php +++ b/Classes/Connection/FacetOptionInterface.php @@ -28,15 +28,11 @@ interface FacetOptionInterface /** * Returns the name of this option. Equivalent * to value used for filtering. - * - * @return string */ - public function getName(); + public function getName() : string; /** * Returns the number of found results for this option. - * - * @return int */ - public function getCount(); + public function getCount() : int; } diff --git a/Classes/Connection/SearchRequestInterface.php b/Classes/Connection/SearchRequestInterface.php index e24adc3..051f32d 100644 --- a/Classes/Connection/SearchRequestInterface.php +++ b/Classes/Connection/SearchRequestInterface.php @@ -20,6 +20,8 @@ namespace Codappix\SearchCore\Connection; * 02110-1301, USA. */ +use Codappix\SearchCore\Connection\ConnectionInterface; +use Codappix\SearchCore\Connection\FacetRequestInterface; use Codappix\SearchCore\Domain\Search\SearchService; use TYPO3\CMS\Extbase\Persistence\QueryInterface; @@ -27,20 +29,27 @@ interface SearchRequestInterface extends QueryInterface { /** * Returns the actual string the user searched for. - * - * @return string */ - public function getSearchTerm(); + public function getSearchTerm() : string; + + public function hasFilter() : bool; + + public function getFilter() : array; + + public function setFilter(array $filter); + + public function addFacet(FacetRequestInterface $facet); /** - * @return bool + * @return array */ - public function hasFilter(); + public function getFacets() : array; /** - * @return array + * Workaround for paginate widget support which will + * use the request to build another search. */ - public function getFilter(); + public function setConnection(ConnectionInterface $connection); /** * Workaround for paginate widget support which will diff --git a/Classes/Connection/SearchResultInterface.php b/Classes/Connection/SearchResultInterface.php index 60718cd..be698fa 100644 --- a/Classes/Connection/SearchResultInterface.php +++ b/Classes/Connection/SearchResultInterface.php @@ -30,19 +30,17 @@ interface SearchResultInterface extends \Iterator, \Countable, QueryResultInterf /** * @return array */ - public function getResults(); + public function getResults() : array; /** * Return all facets, if any. * * @return array */ - public function getFacets(); + public function getFacets() : array; /** * Returns the number of results in current result - * - * @return int */ - public function getCurrentCount(); + public function getCurrentCount() : int; } diff --git a/Classes/Domain/Index/AbstractIndexer.php b/Classes/Domain/Index/AbstractIndexer.php index 2de7dd5..5687cb1 100644 --- a/Classes/Domain/Index/AbstractIndexer.php +++ b/Classes/Domain/Index/AbstractIndexer.php @@ -64,7 +64,7 @@ abstract class AbstractIndexer implements IndexerInterface $this->logger = $logManager->getLogger(__CLASS__); } - public function setIdentifier($identifier) + public function setIdentifier(string $identifier) { $this->identifier = $identifier; } @@ -97,11 +97,11 @@ abstract class AbstractIndexer implements IndexerInterface $this->logger->info('Finish indexing'); } - public function indexDocument($identifier) + public function indexDocument(string $identifier) { $this->logger->info('Start indexing single record.', [$identifier]); try { - $record = $this->getRecord($identifier); + $record = $this->getRecord((int) $identifier); $this->prepareRecord($record); $this->connection->addDocument($this->getDocumentName(), $record); @@ -119,10 +119,7 @@ abstract class AbstractIndexer implements IndexerInterface $this->logger->info('Finish deletion.'); } - /** - * @return \Generator - */ - protected function getRecordGenerator() + protected function getRecordGenerator() : \Generator { $offset = 0; $limit = $this->getLimit(); @@ -133,9 +130,6 @@ abstract class AbstractIndexer implements IndexerInterface } } - /** - * @param array &$record - */ protected function prepareRecord(array &$record) { try { @@ -149,9 +143,6 @@ abstract class AbstractIndexer implements IndexerInterface $this->handleAbstract($record); } - /** - * @param array &$record - */ protected function handleAbstract(array &$record) { $record['search_abstract'] = ''; @@ -177,31 +168,22 @@ abstract class AbstractIndexer implements IndexerInterface /** * Returns the limit to use to fetch records. - * - * @return int */ - protected function getLimit() + protected function getLimit() : int { // TODO: Make configurable. return 50; } /** - * @param int $offset - * @param int $limit * @return array|null */ - abstract protected function getRecords($offset, $limit); + abstract protected function getRecords(int $offset, int $limit); /** - * @param int $identifier - * @return array * @throws NoRecordFoundException If record could not be found. */ - abstract protected function getRecord($identifier); + abstract protected function getRecord(int $identifier) : array; - /** - * @return string - */ - abstract protected function getDocumentName(); + abstract protected function getDocumentName() : string; } diff --git a/Classes/Domain/Index/IndexerFactory.php b/Classes/Domain/Index/IndexerFactory.php index dbae818..6efe035 100644 --- a/Classes/Domain/Index/IndexerFactory.php +++ b/Classes/Domain/Index/IndexerFactory.php @@ -56,12 +56,9 @@ class IndexerFactory implements Singleton } /** - * @param string $identifier - * - * @return IndexerInterface * @throws NoMatchingIndexer */ - public function getIndexer($identifier) + public function getIndexer(string $identifier) : IndexerInterface { try { return $this->buildIndexer($this->configuration->get('indexing.' . $identifier . '.indexer'), $identifier); @@ -75,13 +72,9 @@ class IndexerFactory implements Singleton } /** - * @param string $indexerClass - * @param string $identifier - * - * @return IndexerInterface * @throws NoMatchingIndexer */ - protected function buildIndexer($indexerClass, $identifier) + protected function buildIndexer(string $indexerClass, string $identifier) : IndexerInterface { $indexer = null; if (is_subclass_of($indexerClass, TcaIndexer\PagesIndexer::class) diff --git a/Classes/Domain/Index/IndexerInterface.php b/Classes/Domain/Index/IndexerInterface.php index 72ebb9d..4acfb28 100644 --- a/Classes/Domain/Index/IndexerInterface.php +++ b/Classes/Domain/Index/IndexerInterface.php @@ -35,20 +35,16 @@ interface IndexerInterface /** * Fetches a single document and pushes it to the connection. * - * @param string $identifier - * * @return void */ - public function indexDocument($identifier); + public function indexDocument(string $identifier); /** * Recieves the identifier of the indexer itself. * - * @param string $identifier - * * @return void */ - public function setIdentifier($identifier); + public function setIdentifier(string $identifier); /** * Delete the whole index. diff --git a/Classes/Domain/Index/TcaIndexer.php b/Classes/Domain/Index/TcaIndexer.php index c35b9ea..477bf18 100644 --- a/Classes/Domain/Index/TcaIndexer.php +++ b/Classes/Domain/Index/TcaIndexer.php @@ -22,6 +22,7 @@ namespace Codappix\SearchCore\Domain\Index; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Connection\ConnectionInterface; +use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService; use TYPO3\CMS\Core\Database\ConnectionPool; use TYPO3\CMS\Core\Database\Query\QueryBuilder; use TYPO3\CMS\Core\Utility\GeneralUtility; @@ -46,17 +47,14 @@ class TcaIndexer extends AbstractIndexer ConnectionInterface $connection, ConfigurationContainerInterface $configuration ) { + parent::__construct($connection, $configuration); $this->tcaTableService = $tcaTableService; - $this->connection = $connection; - $this->configuration = $configuration; } /** - * @param int $offset - * @param int $limit * @return array|null */ - protected function getRecords($offset, $limit) + protected function getRecords(int $offset, int $limit) { $records = $this->getQuery() ->setFirstResult($offset) @@ -77,14 +75,12 @@ class TcaIndexer extends AbstractIndexer } /** - * @param int $identifier - * @return array * @throws NoRecordFoundException If record could not be found. */ - protected function getRecord($identifier) + protected function getRecord(int $identifier) : array { $query = $this->getQuery(); - $query = $query->andWhere($this->tcaTableService->getTableName() . '.uid = ' . (int) $identifier); + $query = $query->andWhere($this->tcaTableService->getTableName() . '.uid = ' . $identifier); $record = $query->execute()->fetch(); if ($record === false || $record === null) { @@ -98,15 +94,12 @@ class TcaIndexer extends AbstractIndexer return $record; } - /** - * @return string - */ - protected function getDocumentName() + protected function getDocumentName() : string { return $this->tcaTableService->getTableName(); } - protected function getQuery($tcaTableService = null) : QueryBuilder + protected function getQuery(TcaTableService $tcaTableService = null) : QueryBuilder { if ($tcaTableService === null) { $tcaTableService = $this->tcaTableService; @@ -126,7 +119,7 @@ class TcaIndexer extends AbstractIndexer return $query; } - protected function getDatabaseConnection() + protected function getDatabaseConnection() : ConnectionPool { return GeneralUtility::makeInstance(ConnectionPool::class); } diff --git a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php index b396b73..4f6d03d 100644 --- a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php +++ b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php @@ -36,7 +36,7 @@ class PagesIndexer extends TcaIndexer /** * @param TcaTableService $tcaTableService - * @param TcaTableService $tcaTableService + * @param TcaTableService $contentTableService * @param ConnectionInterface $connection * @param ConfigurationContainerInterface $configuration */ @@ -46,15 +46,10 @@ class PagesIndexer extends TcaIndexer ConnectionInterface $connection, ConfigurationContainerInterface $configuration ) { - $this->tcaTableService = $tcaTableService; + parent::__construct($tcaTableService, $connection, $configuration); $this->contentTableService = $contentTableService; - $this->connection = $connection; - $this->configuration = $configuration; } - /** - * @param array &$record - */ protected function prepareRecord(array &$record) { $possibleTitleFields = ['nav_title', 'tx_tqseo_pagetitle_rel', 'title']; @@ -69,11 +64,7 @@ class PagesIndexer extends TcaIndexer parent::prepareRecord($record); } - /** - * @param int $uid - * @return string - */ - protected function fetchContentForPage($uid) + protected function fetchContentForPage(int $uid) : string { $contentElements = $this->getQuery($this->contentTableService)->execute()->fetchAll(); diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index f39248f..61e30a9 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -107,17 +107,11 @@ class TcaTableService $this->relationResolver = $relationResolver; } - /** - * @return string - */ public function getTableName() : string { return $this->tableName; } - /** - * @return string - */ public function getTableClause() : string { return $this->tableName; @@ -125,9 +119,6 @@ class TcaTableService /** * Filter the given records by root line blacklist settings. - * - * @param array &$records - * @return void */ public function filterRecordsByRootLineBlacklist(array &$records) { @@ -139,9 +130,6 @@ class TcaTableService ); } - /** - * @param array &$record - */ public function prepareRecord(array &$record) { $this->relationResolver->resolveRelationsForRecord($this, $record); @@ -166,8 +154,8 @@ class TcaTableService $whereClause .= ' AND ' . $userDefinedWhere; } - if ($this->isBlacklistedRootLineConfigured()) { - $parameters[':blacklistedRootLine'] = $this->getBlacklistedRootLine(); + if ($this->isBlackListedRootLineConfigured()) { + $parameters[':blacklistedRootLine'] = $this->getBlackListedRootLine(); $whereClause .= ' AND pages.uid NOT IN (:blacklistedRootLine)' . ' AND pages.pid NOT IN (:blacklistedRootLine)'; } @@ -231,11 +219,7 @@ class TcaTableService return $whereClause; } - /** - * @param string - * @return bool - */ - protected function isSystemField($columnName) : bool + protected function isSystemField(string $columnName) : bool { $systemFields = [ // Versioning fields, @@ -267,11 +251,9 @@ class TcaTableService } /** - * @param string $columnName - * @return array * @throws InvalidArgumentException */ - public function getColumnConfig($columnName) : array + public function getColumnConfig(string $columnName) : array { if (!isset($this->tca['columns'][$columnName])) { throw new InvalidArgumentException( @@ -290,9 +272,6 @@ class TcaTableService * Also further TYPO3 mechanics are taken into account. Does a valid root * line exist, is page inside a recycler, is inherited start- endtime * excluded, etc. - * - * @param array &$record - * @return bool */ protected function isRecordBlacklistedByRootline(array &$record) : bool { @@ -348,8 +327,6 @@ class TcaTableService /** * Checks whether any page uids are black listed. - * - * @return bool */ protected function isBlackListedRootLineConfigured() : bool { diff --git a/Classes/Domain/Model/SearchRequest.php b/Classes/Domain/Model/SearchRequest.php index 69237bb..2addff1 100644 --- a/Classes/Domain/Model/SearchRequest.php +++ b/Classes/Domain/Model/SearchRequest.php @@ -72,56 +72,39 @@ class SearchRequest implements SearchRequestInterface /** * @param string $query */ - public function __construct($query = '') + public function __construct(string $query = '') { - $this->query = (string) $query; + $this->query = $query; } - /** - * @return string - */ - public function getQuery() + public function getQuery() : string { return $this->query; } - /** - * @return string - */ - public function getSearchTerm() + public function getSearchTerm() : string { return $this->query; } - /** - * @param array $filter - */ public function setFilter(array $filter) { $filter = \TYPO3\CMS\Core\Utility\ArrayUtility::removeArrayEntryByValue($filter, ''); $this->filter = \TYPO3\CMS\Extbase\Utility\ArrayUtility::removeEmptyElementsRecursively($filter); } - /** - * @return bool - */ - public function hasFilter() + public function hasFilter() : bool { return count($this->filter) > 0; } - /** - * @return array - */ - public function getFilter() + public function getFilter() : array { return $this->filter; } /** * Add a facet to gather in this search request. - * - * @param FacetRequestInterface $facet */ public function addFacet(FacetRequestInterface $facet) { @@ -130,10 +113,8 @@ class SearchRequest implements SearchRequestInterface /** * Returns all configured facets to fetch in this search request. - * - * @return array */ - public function getFacets() + public function getFacets() : array { return $this->facets; } @@ -141,8 +122,6 @@ class SearchRequest implements SearchRequestInterface /** * Define connection to use for this request. * Necessary to allow implementation of execute for interface. - * - * @param ConnectionInterface $connection */ public function setConnection(ConnectionInterface $connection) { diff --git a/Classes/Domain/Model/SearchResult.php b/Classes/Domain/Model/SearchResult.php index d91820d..163b996 100644 --- a/Classes/Domain/Model/SearchResult.php +++ b/Classes/Domain/Model/SearchResult.php @@ -62,7 +62,7 @@ class SearchResult implements SearchResultInterface /** * @return array */ - public function getResults() + public function getResults() : array { $this->initResults(); @@ -80,12 +80,12 @@ class SearchResult implements SearchResultInterface } } - public function getFacets() + public function getFacets() : array { return $this->originalSearchResult->getFacets(); } - public function getCurrentCount() + public function getCurrentCount() : int { return $this->originalSearchResult->getCurrentCount(); } diff --git a/Classes/Domain/Search/SearchService.php b/Classes/Domain/Search/SearchService.php index 03fc11b..3a83b7b 100644 --- a/Classes/Domain/Search/SearchService.php +++ b/Classes/Domain/Search/SearchService.php @@ -74,11 +74,7 @@ class SearchService $this->dataProcessorService = $dataProcessorService; } - /** - * @param SearchRequestInterface $searchRequest - * @return SearchResultInterface - */ - public function search(SearchRequestInterface $searchRequest) + public function search(SearchRequestInterface $searchRequest) : SearchResultInterface { $this->addSize($searchRequest); $this->addConfiguredFacets($searchRequest); @@ -93,8 +89,6 @@ class SearchService /** * Add configured size of search result items to request. - * - * @param SearchRequestInterface $searchRequest */ protected function addSize(SearchRequestInterface $searchRequest) { @@ -105,8 +99,6 @@ class SearchService /** * Add facets from configuration to request. - * - * @param SearchRequestInterface $searchRequest */ protected function addConfiguredFacets(SearchRequestInterface $searchRequest) { @@ -126,8 +118,6 @@ class SearchService /** * Add filters from configuration, e.g. flexform or TypoScript. - * - * @param SearchRequestInterface $searchRequest */ protected function addConfiguredFilters(SearchRequestInterface $searchRequest) { diff --git a/Classes/Domain/Service/DataHandler.php b/Classes/Domain/Service/DataHandler.php index 10a01fe..fd6b08d 100644 --- a/Classes/Domain/Service/DataHandler.php +++ b/Classes/Domain/Service/DataHandler.php @@ -22,6 +22,7 @@ namespace Codappix\SearchCore\Domain\Service; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Domain\Index\IndexerFactory; +use Codappix\SearchCore\Domain\Index\IndexerInterface; use Codappix\SearchCore\Domain\Index\NoMatchingIndexerException; use Codappix\SearchCore\Domain\Index\TcaIndexer; use TYPO3\CMS\Core\SingletonInterface as Singleton; @@ -83,41 +84,27 @@ class DataHandler implements Singleton $this->indexerFactory = $indexerFactory; } - /** - * @param string $table - */ - public function update($table, array $record) + public function update(string $table, array $record) { $this->logger->debug('Record received for update.', [$table, $record]); $this->getIndexer($table)->indexDocument($record['uid']); } - /** - * @param string $table - * @param int $identifier - */ - public function delete($table, $identifier) + public function delete(string $table, string $identifier) { $this->logger->debug('Record received for delete.', [$table, $identifier]); $this->connection->deleteDocument($table, $identifier); } /** - * @param string $table - * @return IndexerInterface - * * @throws NoMatchingIndexerException */ - protected function getIndexer($table) + protected function getIndexer(string $table) : IndexerInterface { return $this->indexerFactory->getIndexer($table); } - /** - * @param string $table - * @return bool - */ - public function canHandle($table) + public function canHandle(string $table) : bool { try { $this->getIndexer($table); diff --git a/Classes/Hook/DataHandler.php b/Classes/Hook/DataHandler.php index fa4b09f..6277708 100644 --- a/Classes/Hook/DataHandler.php +++ b/Classes/Hook/DataHandler.php @@ -49,46 +49,38 @@ class DataHandler implements Singleton /** * Dependency injection as TYPO3 doesn't provide it on it's own. * Still you can submit your own dataHandler. - * - * @param OwnDataHandler $dataHandler - * @param Logger $logger */ public function __construct(OwnDataHandler $dataHandler = null, Logger $logger = null) { - $this->dataHandler = $dataHandler; - if ($this->dataHandler === null) { + if ($dataHandler === null) { try { - $this->dataHandler = GeneralUtility::makeInstance(ObjectManager::class) + $dataHandler = GeneralUtility::makeInstance(ObjectManager::class) ->get(OwnDataHandler::class); } catch (NoConfigurationException $e) { // We have no configuration. That's fine, hooks will not be // executed due to check for existing DataHandler. } } + $this->dataHandler = $dataHandler; - $this->logger = $logger; - if ($this->logger === null) { - $this->logger = GeneralUtility::makeInstance(LogManager::class) + if ($logger === null) { + $logger = GeneralUtility::makeInstance(LogManager::class) ->getLogger(__CLASS__); } + $this->logger = $logger; } /** * Called by CoreDataHandler on deletion of records. - * - * @param string $table - * @param int $uid - * - * @return bool False if hook was not processed. */ - public function processCmdmap_deleteAction($table, $uid) + public function processCmdmap_deleteAction(string $table, int $uid) : bool { if (! $this->shouldProcessHookForTable($table)) { $this->logger->debug('Delete not processed.', [$table, $uid]); return false; } - $this->dataHandler->delete($table, $uid); + $this->dataHandler->delete($table, (string) $uid); return true; } @@ -125,11 +117,7 @@ class DataHandler implements Singleton return false; } - /** - * @param string $table - * @return bool - */ - protected function shouldProcessHookForTable($table) + protected function shouldProcessHookForTable(string $table) : bool { if ($this->dataHandler === null) { $this->logger->debug('Datahandler could not be setup.'); @@ -146,11 +134,9 @@ class DataHandler implements Singleton /** * Wrapper to allow unit testing. * - * @param string $table - * @param int $uid - * @return null|array + * @return array|null */ - protected function getRecord($table, $uid) + protected function getRecord(string $table, int $uid) { return BackendUtility::getRecord($table, $uid); } diff --git a/Classes/Integration/Form/Finisher/DataHandlerFinisher.php b/Classes/Integration/Form/Finisher/DataHandlerFinisher.php index 696d8ad..6a90e01 100644 --- a/Classes/Integration/Form/Finisher/DataHandlerFinisher.php +++ b/Classes/Integration/Form/Finisher/DataHandlerFinisher.php @@ -62,7 +62,7 @@ class DataHandlerFinisher extends AbstractFinisher $this->dataHandler->update($tableName, $record); break; case 'delete': - $this->dataHandler->delete($tableName, $record['uid']); + $this->dataHandler->delete($tableName, (string) $record['uid']); break; } } diff --git a/Classes/Utility/FrontendUtility.php b/Classes/Utility/FrontendUtility.php index 1282421..ffdbb6d 100644 --- a/Classes/Utility/FrontendUtility.php +++ b/Classes/Utility/FrontendUtility.php @@ -29,10 +29,7 @@ use TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController; */ class FrontendUtility extends BackendUtility { - /** - * @return TypoScriptFrontendController - */ - protected static function getLanguageService() + protected static function getLanguageService() : TypoScriptFrontendController { return $GLOBALS['TSFE']; } From 8d930448e0d1b63c0c8bfd30c7b0b556907ae253 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 17:58:19 +0100 Subject: [PATCH 25/75] TASK: Fix scurtinizer issues --- Classes/Connection/SearchRequestInterface.php | 7 +++++++ Classes/DataProcessing/GeoPointProcessor.php | 4 ++-- Classes/Domain/Index/AbstractIndexer.php | 2 +- Classes/Domain/Service/DataHandler.php | 4 +--- Classes/Hook/DataHandler.php | 6 +++--- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/Classes/Connection/SearchRequestInterface.php b/Classes/Connection/SearchRequestInterface.php index 051f32d..a400447 100644 --- a/Classes/Connection/SearchRequestInterface.php +++ b/Classes/Connection/SearchRequestInterface.php @@ -38,6 +38,9 @@ interface SearchRequestInterface extends QueryInterface public function setFilter(array $filter); + /** + * @return void + */ public function addFacet(FacetRequestInterface $facet); /** @@ -48,12 +51,16 @@ interface SearchRequestInterface extends QueryInterface /** * Workaround for paginate widget support which will * use the request to build another search. + * + * @return void */ public function setConnection(ConnectionInterface $connection); /** * Workaround for paginate widget support which will * use the request to build another search. + * + * @return void */ public function setSearchService(SearchService $searchService); } diff --git a/Classes/DataProcessing/GeoPointProcessor.php b/Classes/DataProcessing/GeoPointProcessor.php index c5b6d18..971e2c4 100644 --- a/Classes/DataProcessing/GeoPointProcessor.php +++ b/Classes/DataProcessing/GeoPointProcessor.php @@ -27,7 +27,7 @@ class GeoPointProcessor implements ProcessorInterface { public function processData(array $record, array $configuration) : array { - if (! $this->canApply($record, $configuration)) { + if (! $this->isApplyable($record, $configuration)) { return $record; } @@ -39,7 +39,7 @@ class GeoPointProcessor implements ProcessorInterface return $record; } - protected function canApply(array $record, array $configuration) : bool + protected function isApplyable(array $record, array $configuration) : bool { if (!isset($record[$configuration['lat']]) || !is_numeric($record[$configuration['lat']]) diff --git a/Classes/Domain/Index/AbstractIndexer.php b/Classes/Domain/Index/AbstractIndexer.php index 5687cb1..e8f032e 100644 --- a/Classes/Domain/Index/AbstractIndexer.php +++ b/Classes/Domain/Index/AbstractIndexer.php @@ -152,7 +152,7 @@ abstract class AbstractIndexer implements IndexerInterface ',', $this->configuration->get('indexing.' . $this->identifier . '.abstractFields') ); - if (!$fieldsToUse) { + if ($fieldsToUse === []) { return; } foreach ($fieldsToUse as $fieldToUse) { diff --git a/Classes/Domain/Service/DataHandler.php b/Classes/Domain/Service/DataHandler.php index fd6b08d..5b587e4 100644 --- a/Classes/Domain/Service/DataHandler.php +++ b/Classes/Domain/Service/DataHandler.php @@ -104,7 +104,7 @@ class DataHandler implements Singleton return $this->indexerFactory->getIndexer($table); } - public function canHandle(string $table) : bool + public function supportsTable(string $table) : bool { try { $this->getIndexer($table); @@ -112,7 +112,5 @@ class DataHandler implements Singleton } catch (NoMatchingIndexerException $e) { return false; } - - return false; } } diff --git a/Classes/Hook/DataHandler.php b/Classes/Hook/DataHandler.php index 6277708..3bd698f 100644 --- a/Classes/Hook/DataHandler.php +++ b/Classes/Hook/DataHandler.php @@ -90,8 +90,8 @@ class DataHandler implements Singleton $uid = key($record); $fieldData = current($record); - if (isset($fieldArray['uid'])) { - $uid = $fieldArray['uid']; + if (isset($fieldData['uid'])) { + $uid = $fieldData['uid']; } elseif (isset($dataHandler->substNEWwithIDs[$uid])) { $uid = $dataHandler->substNEWwithIDs[$uid]; } @@ -123,7 +123,7 @@ class DataHandler implements Singleton $this->logger->debug('Datahandler could not be setup.'); return false; } - if (! $this->dataHandler->canHandle($table)) { + if (! $this->dataHandler->supportsTable($table)) { $this->logger->debug('Table is not allowed.', [$table]); return false; } From 9a2d2734f29e319fd24c615c862480325c20df26 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 17:58:27 +0100 Subject: [PATCH 26/75] TASK: Switch to new php scurtinizer engine --- .scrutinizer.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.scrutinizer.yml b/.scrutinizer.yml index ca92884..3122a2f 100644 --- a/.scrutinizer.yml +++ b/.scrutinizer.yml @@ -1,3 +1,11 @@ +build: + nodes: + analysis: + project_setup: + override: true + tests: + override: [php-scrutinizer-run] + filter: excluded_paths: - 'Configuration/*' @@ -19,7 +27,7 @@ tools: php_hhvm: enabled: true config: - use_undeclared_constant: false + use_undeclared_constant: false php_mess_detector: enabled: true @@ -34,5 +42,5 @@ tools: enabled: true # We generate code coverage during tests at travis and will send them here external_code_coverage: - runs: 2 - timeout: 1200 + runs: 2 + timeout: 1200 From 30a833ce68c5af08a97667a6e0b35b06ccb3aaa0 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 8 Mar 2018 08:23:37 +0100 Subject: [PATCH 27/75] TASK: Add branch alias to provide version number Allow to require extension with 1.0.0, if dev is allowed. --- composer.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/composer.json b/composer.json index a630385..f58a062 100644 --- a/composer.json +++ b/composer.json @@ -37,6 +37,9 @@ ] }, "extra": { + "branch-alias": { + "dev-develop": "1.0.x-dev" + }, "typo3/cms": { "cms-package-dir": "{$vendor-dir}/typo3/cms", "web-dir": ".Build/web" From e960f4595b5727ec7ab3e870c7a9eedac611cda2 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 8 Mar 2018 09:54:34 +0100 Subject: [PATCH 28/75] TASK: Add breaking change documentation Also order changelog entries news first. --- Documentation/source/changelog.rst | 3 ++- .../20180408-introduce-php70-type-hints.rst | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 Documentation/source/changelog/20180408-introduce-php70-type-hints.rst diff --git a/Documentation/source/changelog.rst b/Documentation/source/changelog.rst index 0d7f020..3b121a3 100644 --- a/Documentation/source/changelog.rst +++ b/Documentation/source/changelog.rst @@ -5,4 +5,5 @@ Changelog :maxdepth: 1 :glob: - changelog/* + changelog/20180408-introduce-php70-type-hints + changelog/20180406-120-facet-configuration diff --git a/Documentation/source/changelog/20180408-introduce-php70-type-hints.rst b/Documentation/source/changelog/20180408-introduce-php70-type-hints.rst new file mode 100644 index 0000000..cc2dfe3 --- /dev/null +++ b/Documentation/source/changelog/20180408-introduce-php70-type-hints.rst @@ -0,0 +1,12 @@ +Breacking Change "Introduce PHP 7.0 TypeHints" +============================================== + +As PHP evolved, we now migrate the whole code base to use PHP 7.0 type hints. +We do not use PHP 7.1 Type Hints, as some customers still need PHP 7.0 support. + +Also we added missing methods to interfaces, that were already used in code. + +As this leads to changed method signatures, most custom implementations of interfaces, or overwrites +of existing methods are broken. + +To fix, just update the signatures as pointed out by PHP while running the code. From 43f31f75a8c7a6edbe3e1e7fc7f4baa265b52204 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 8 Mar 2018 11:08:18 +0100 Subject: [PATCH 29/75] TASK: Add phan configuration Even if not used in CI yet, we still can use it local. --- .phan/config.php | 96 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 .phan/config.php diff --git a/.phan/config.php b/.phan/config.php new file mode 100644 index 0000000..11a17af --- /dev/null +++ b/.phan/config.php @@ -0,0 +1,96 @@ + '7.0', + + // Override to hardcode existence and types of (non-builtin) globals. + // Class names should be prefixed with '\\'. + // (E.g. ['_FOO' => '\\FooClass', 'page' => '\\PageClass', 'userId' => 'int']) + 'globals_type_map' => [ + '_EXTKEY' => 'string', + 'EM_CONF' => 'array', + ], + + // A list of directories that should be parsed for class and + // method information. After excluding the directories + // defined in exclude_analysis_directory_list, the remaining + // files will be statically analyzed for errors. + // + // Thus, both first-party and third-party code being used by + // your application should be included in this list. + 'directory_list' => [ + 'Classes', + '.Build/vendor', + ], + + // A list of files to include in analysis + 'file_list' => [ + 'ext_emconf.php', + 'ext_tables.php', + 'ext_localconf.php', + ], + + // A directory list that defines files that will be excluded + // from static analysis, but whose class and method + // information should be included. + // + // Generally, you'll want to include the directories for + // third-party code (such as "vendor/") in this list. + // + // n.b.: If you'd like to parse but not analyze 3rd + // party code, directories containing that code + // should be added to the `directory_list` as + // to `exclude_analysis_directory_list`. + "exclude_analysis_directory_list" => [ + '.Build/vendor' + ], + + // A list of directories that should be parsed for class and + // method information. After excluding the directories + // defined in exclude_analysis_directory_list, the remaining + // files will be statically analyzed for errors. + // + // Thus, both first-party and third-party code being used by + // your application should be included in this list. + 'directory_list' => [ + 'Classes', + // 'Tests', + '.Build/vendor', + ], + + // The number of processes to fork off during the analysis phase. + 'processes' => 3, + + // Add any issue types (such as 'PhanUndeclaredMethod') + // here to inhibit them from being reported + 'suppress_issue_types' => [ + 'PhanDeprecatedFunction', // For now + 'PhanParamTooMany', // For now, due to ObjectManager->get() + ], + + // A list of plugin files to execute. + // See https://github.com/phan/phan/tree/master/.phan/plugins for even more. + // (Pass these in as relative paths. + // The 0.10.2 release will allow passing 'AlwaysReturnPlugin' if referring to a plugin that is bundled with Phan) + 'plugins' => [ + // checks if a function, closure or method unconditionally returns. + 'AlwaysReturnPlugin', // can also be written as 'vendor/phan/phan/.phan/plugins/AlwaysReturnPlugin.php' + // Checks for syntactically unreachable statements in + // the global scope or function bodies. + 'UnreachableCodePlugin', + 'DollarDollarPlugin', + 'DuplicateArrayKeyPlugin', + 'PregRegexCheckerPlugin', + 'PrintfCheckerPlugin', + ], +]; From 4da9e86540bd0864651d70f563dc5841e9d3f730 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 8 Mar 2018 11:09:07 +0100 Subject: [PATCH 30/75] TASK: Remove unused use statements --- Classes/Command/IndexCommandController.php | 1 - Classes/Connection/Elasticsearch/IndexFactory.php | 2 -- Classes/Connection/Elasticsearch/TypeFactory.php | 1 - Classes/Domain/Index/AbstractIndexer.php | 1 - Classes/Domain/Search/QueryFactory.php | 2 -- Classes/Domain/Service/DataHandler.php | 2 -- Classes/Hook/DataHandler.php | 1 - 7 files changed, 10 deletions(-) diff --git a/Classes/Command/IndexCommandController.php b/Classes/Command/IndexCommandController.php index 61aad53..e0bdeb9 100644 --- a/Classes/Command/IndexCommandController.php +++ b/Classes/Command/IndexCommandController.php @@ -22,7 +22,6 @@ namespace Codappix\SearchCore\Command; use Codappix\SearchCore\Domain\Index\IndexerFactory; use Codappix\SearchCore\Domain\Index\NoMatchingIndexerException; -use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Extbase\Mvc\Controller\CommandController; /** diff --git a/Classes/Connection/Elasticsearch/IndexFactory.php b/Classes/Connection/Elasticsearch/IndexFactory.php index 6ddab67..8b57783 100644 --- a/Classes/Connection/Elasticsearch/IndexFactory.php +++ b/Classes/Connection/Elasticsearch/IndexFactory.php @@ -22,10 +22,8 @@ namespace Codappix\SearchCore\Connection\Elasticsearch; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\InvalidArgumentException; -use Elastica\Exception\ResponseException; use TYPO3\CMS\Core\SingletonInterface as Singleton; use TYPO3\CMS\Core\Utility\GeneralUtility; -use TYPO3\CMS\Extbase\Configuration\ConfigurationManagerInterface; /** * Factory to get indexes. diff --git a/Classes/Connection/Elasticsearch/TypeFactory.php b/Classes/Connection/Elasticsearch/TypeFactory.php index de8e336..e84cdd0 100644 --- a/Classes/Connection/Elasticsearch/TypeFactory.php +++ b/Classes/Connection/Elasticsearch/TypeFactory.php @@ -21,7 +21,6 @@ namespace Codappix\SearchCore\Connection\Elasticsearch; */ use TYPO3\CMS\Core\SingletonInterface as Singleton; -use TYPO3\CMS\Extbase\Configuration\ConfigurationManagerInterface; /** * Factory to get indexes. diff --git a/Classes/Domain/Index/AbstractIndexer.php b/Classes/Domain/Index/AbstractIndexer.php index e8f032e..d264b08 100644 --- a/Classes/Domain/Index/AbstractIndexer.php +++ b/Classes/Domain/Index/AbstractIndexer.php @@ -23,7 +23,6 @@ namespace Codappix\SearchCore\Domain\Index; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Connection\ConnectionInterface; -use Codappix\SearchCore\DataProcessing\ProcessorInterface; use TYPO3\CMS\Core\Utility\GeneralUtility; abstract class AbstractIndexer implements IndexerInterface diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index dcf6ce5..80c5151 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -23,8 +23,6 @@ namespace Codappix\SearchCore\Domain\Search; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\ConfigurationUtility; use Codappix\SearchCore\Configuration\InvalidArgumentException; -use Codappix\SearchCore\Connection\ConnectionInterface; -use Codappix\SearchCore\Connection\Elasticsearch\Query; use Codappix\SearchCore\Connection\SearchRequestInterface; use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Extbase\Utility\ArrayUtility; diff --git a/Classes/Domain/Service/DataHandler.php b/Classes/Domain/Service/DataHandler.php index 5b587e4..de226b9 100644 --- a/Classes/Domain/Service/DataHandler.php +++ b/Classes/Domain/Service/DataHandler.php @@ -24,9 +24,7 @@ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Domain\Index\IndexerFactory; use Codappix\SearchCore\Domain\Index\IndexerInterface; use Codappix\SearchCore\Domain\Index\NoMatchingIndexerException; -use Codappix\SearchCore\Domain\Index\TcaIndexer; use TYPO3\CMS\Core\SingletonInterface as Singleton; -use TYPO3\CMS\Core\Utility\GeneralUtility; /** * Handles all data related things like updates, deletes and inserts. diff --git a/Classes/Hook/DataHandler.php b/Classes/Hook/DataHandler.php index 3bd698f..71fe44b 100644 --- a/Classes/Hook/DataHandler.php +++ b/Classes/Hook/DataHandler.php @@ -21,7 +21,6 @@ namespace Codappix\SearchCore\Hook; */ use Codappix\SearchCore\Configuration\NoConfigurationException; -use Codappix\SearchCore\Domain\Index\NoMatchingIndexerException; use Codappix\SearchCore\Domain\Service\DataHandler as OwnDataHandler; use TYPO3\CMS\Backend\Utility\BackendUtility; use TYPO3\CMS\Core\DataHandling\DataHandler as CoreDataHandler; From fc8017fdf41a4e635baf7c635aed2bf1d5688087 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 8 Mar 2018 11:09:28 +0100 Subject: [PATCH 31/75] TASK: Fix further issues reported by phan --- Classes/Connection/Elasticsearch/Connection.php | 2 +- .../ContentObjectDataProcessorAdapterProcessor.php | 2 +- Classes/Domain/Index/TcaIndexer/TcaTableService.php | 2 +- Classes/Domain/Model/SearchRequest.php | 4 ++++ 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Classes/Connection/Elasticsearch/Connection.php b/Classes/Connection/Elasticsearch/Connection.php index 10a62d3..23ae703 100644 --- a/Classes/Connection/Elasticsearch/Connection.php +++ b/Classes/Connection/Elasticsearch/Connection.php @@ -44,7 +44,7 @@ class Connection implements Singleton /** * @param ConfigurationContainerInterface $configuration - * @param \Elastica\Client $elasticaClient + * @param \Elastica\Client|null $elasticaClient */ public function __construct( ConfigurationContainerInterface $configuration, diff --git a/Classes/DataProcessing/ContentObjectDataProcessorAdapterProcessor.php b/Classes/DataProcessing/ContentObjectDataProcessorAdapterProcessor.php index 322486b..c3b4c32 100644 --- a/Classes/DataProcessing/ContentObjectDataProcessorAdapterProcessor.php +++ b/Classes/DataProcessing/ContentObjectDataProcessorAdapterProcessor.php @@ -21,7 +21,7 @@ namespace Codappix\SearchCore\DataProcessing; */ use TYPO3\CMS\Core\Utility\GeneralUtility; -use TYPO3\CMS\Extbase\Service\TypoScriptService; +use TYPO3\CMS\Core\TypoScript\TypoScriptService; use TYPO3\CMS\Frontend\ContentObject\ContentObjectRenderer; /** diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index 61e30a9..19f6a55 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -21,7 +21,7 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer; */ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; -use Codappix\SearchCore\Configuration\InvalidArgumentException as InvalidConfigurationArgumentException; +use Codappix\SearchCore\Domain\Index\TcaIndexer\InvalidArgumentException; use Codappix\SearchCore\Database\Doctrine\Join; use Codappix\SearchCore\Database\Doctrine\Where; use Codappix\SearchCore\Domain\Index\IndexingException; diff --git a/Classes/Domain/Model/SearchRequest.php b/Classes/Domain/Model/SearchRequest.php index 2addff1..22bd697 100644 --- a/Classes/Domain/Model/SearchRequest.php +++ b/Classes/Domain/Model/SearchRequest.php @@ -156,11 +156,15 @@ class SearchRequest implements SearchRequestInterface public function setLimit($limit) { $this->limit = (int) $limit; + + return $this; } public function setOffset($offset) { $this->offset = (int) $offset; + + return $this; } public function getLimit() From 09cd655a7a2bc0336f9e8b20035d755b31596ffd Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 8 Mar 2018 11:20:23 +0100 Subject: [PATCH 32/75] TASK: Remove whitespace in return type annotation --- Classes/Connection/Elasticsearch/Connection.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Classes/Connection/Elasticsearch/Connection.php b/Classes/Connection/Elasticsearch/Connection.php index 23ae703..a5e7d0f 100644 --- a/Classes/Connection/Elasticsearch/Connection.php +++ b/Classes/Connection/Elasticsearch/Connection.php @@ -68,7 +68,7 @@ class Connection implements Singleton /** * Get the concrete client for internal usage! */ - public function getClient() : \ Elastica\Client + public function getClient() : \Elastica\Client { return $this->elasticaClient; } From 4756da5c5d5df04ed94705d7cbd3b2caecaae3a4 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 8 Mar 2018 08:17:41 +0100 Subject: [PATCH 33/75] TASK: Update / cleanup documentation Code and docs have diverged a bit. We therefore need to update them. --- .../20180406-120-facet-configuration.rst | 4 +- Documentation/source/concepts.rst | 33 ++-- Documentation/source/conf.py | 10 +- Documentation/source/configuration.rst | 8 +- .../source/configuration/connections.rst | 11 +- .../dataProcessing/RemoveProcessor.rst | 2 +- .../dataProcessing/availableAndPlanned.rst | 36 ---- .../source/configuration/indexing.rst | 54 ++---- .../source/configuration/searching.rst | 166 ++++++++---------- Documentation/source/connections.rst | 25 ++- Documentation/source/dataprocessors.rst | 98 +++++++++++ Documentation/source/development.rst | 75 ++------ .../source/development/connection.rst | 11 ++ .../source/development/contribution.rst | 69 ++++++++ .../source/development/dataProcessor.rst | 15 ++ Documentation/source/development/indexer.rst | 21 +++ Documentation/source/features.rst | 40 +++-- Documentation/source/index.rst | 3 +- Documentation/source/indexer.rst | 39 +--- Documentation/source/installation.rst | 28 ++- Documentation/source/readme.rst | 19 +- Documentation/source/usage.rst | 18 +- composer.json | 3 + 23 files changed, 434 insertions(+), 354 deletions(-) delete mode 100644 Documentation/source/configuration/dataProcessing/availableAndPlanned.rst create mode 100644 Documentation/source/dataprocessors.rst create mode 100644 Documentation/source/development/connection.rst create mode 100644 Documentation/source/development/contribution.rst create mode 100644 Documentation/source/development/dataProcessor.rst create mode 100644 Documentation/source/development/indexer.rst diff --git a/Documentation/source/changelog/20180406-120-facet-configuration.rst b/Documentation/source/changelog/20180406-120-facet-configuration.rst index 53021d9..0c2338f 100644 --- a/Documentation/source/changelog/20180406-120-facet-configuration.rst +++ b/Documentation/source/changelog/20180406-120-facet-configuration.rst @@ -1,4 +1,4 @@ -Breacking Change 120 "Pass facets configuration to elasticsearch" +Breacking Change 120 "Pass facets configuration to Elasticsearch" ================================================================= In order to allow arbitrary facet configuration, we do not process the facet configuration anymore. @@ -35,6 +35,6 @@ Instead you have to provide the full configuration yourself: } } -You need to add line 4 and 6, the additional level ``terms`` for elasticsearch. +You need to add line 4 and 6, the additional level ``terms`` for Elasticsearch. See :issue:`120`. diff --git a/Documentation/source/concepts.rst b/Documentation/source/concepts.rst index f4c5bbd..f81121b 100644 --- a/Documentation/source/concepts.rst +++ b/Documentation/source/concepts.rst @@ -3,33 +3,35 @@ Concepts ======== -The extension is built with the following concepts in mind. +The main concept is to provide a foundation where other developers can profit from, to provide +integrations into search services like Elasticsearch, Algolia, ... . + +Our code contains the following concepts which should be understand: .. _concepts_connections: Connections ----------- -It should be possible to use different search services like elasticsearch and solr out of the box. -If a service is not contained, it should be possible to implement the necessary part by implementing -the necessary interfaces and configuring the extension to use the new connection. +Different search services can provide integrations. ``search_core`` only provides abstractions and +interfaces. The main purpose is to provide a stable API between TYPO3 and concrete connection. -Also it should be possible to use multiple connections at once. This way multiple search services -can be used in the same installation. - -Currently only :ref:`Elasticsearch` is provided. +For information about implementing a new connection, take a look at :ref:`development_connection`. .. _concepts_indexing: Indexing -------- -The indexing is done by one of the available indexer. For each identifier it's possible to define -the indexer to use. Also it's possible to write custom indexer to use. +Indexing is the process of collecting and preparing data, before sending it to a Connection. +The indexing is done by one of the available indexer. Indexer are identified by a key, as configured +in TypoScript. -Currently only the :ref:`TcaIndexer` is provided. +Currently :ref:`TcaIndexer` and :ref:`PagesIndexer` are provided. -.. _concepts_indexing_dataprocessing: +For information about implementing a new indexer, take a look at :ref:`development_indexer`. + +.. _concepts_dataprocessing: DataProcessing ^^^^^^^^^^^^^^ @@ -39,4 +41,9 @@ known by :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing` of :ref:`t3 The same is true for retrieved search results. They can be processed again by "DataProcessors" to prepare data for display in Templates or further usage. -Configuration is done through TypoScript, see :ref:`dataProcessing`. +This should keep indexers simple and move logic to DataProcessors. This makes most parts highly +flexible as integrators are able to configure DataProcessors and change their order. + +Configuration is done through TypoScript, see :ref:`dataprocessors`. + +For information about implementing a new DataProcessor, take a look at :ref:`development_dataprocessor`. diff --git a/Documentation/source/conf.py b/Documentation/source/conf.py index 1419689..aa846e0 100644 --- a/Documentation/source/conf.py +++ b/Documentation/source/conf.py @@ -51,7 +51,7 @@ master_doc = 'index' # General information about the project. project = u'TYPO3 Extension search_core' -copyright = u'2016, Daniel Siepmann' +copyright = u'2016 - 2018, Daniel Siepmann' author = u'Daniel Siepmann' # The version info for the project you're documenting, acts as replacement for @@ -59,9 +59,9 @@ author = u'Daniel Siepmann' # built documents. # # The short X.Y version. -version = u'1.0.0' +version = u'0.0.1' # The full version, including alpha/beta/rc tags. -release = u'1.0.0' +release = u'0.0.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -133,7 +133,7 @@ html_theme_options = { # The name for this set of Sphinx documents. # " v documentation" by default. -#html_title = u'TYPO3 Extension search_core v1.0.0' +#html_title = u'TYPO3 Extension search_core v0.0.1' # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None @@ -150,7 +150,7 @@ html_theme_options = { # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +# html_static_path = ['_static'] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied diff --git a/Documentation/source/configuration.rst b/Documentation/source/configuration.rst index 09289ac..53ac469 100644 --- a/Documentation/source/configuration.rst +++ b/Documentation/source/configuration.rst @@ -8,9 +8,9 @@ Configuration Installation wide configuration is handled inside of the extension manager. Just check out the options there, they all have labels. -The extension offers the following configuration options through TypoScript. If you overwrite them -through `setup` make sure to keep them in the `module` area as they will be accessed from backend -mode of TYPO3 for indexing. Do so by placing the following line at the end:: +Everything else is configured through TypoScript. If you overwrite them through `setup` make sure to +keep them in the `module` area as they will be accessed from backend mode of TYPO3 for indexing. Do +so by placing the following line at the end:: module.tx_searchcore < plugin.tx_searchcore @@ -26,12 +26,10 @@ Here is the example default configuration that's provided through static include .. literalinclude:: ../../Configuration/TypoScript/constants.txt :language: typoscript - :linenos: :caption: Static TypoScript Constants .. literalinclude:: ../../Configuration/TypoScript/setup.txt :language: typoscript - :linenos: :caption: Static TypoScript Setup .. _configuration_options: diff --git a/Documentation/source/configuration/connections.rst b/Documentation/source/configuration/connections.rst index 5819730..6d0c524 100644 --- a/Documentation/source/configuration/connections.rst +++ b/Documentation/source/configuration/connections.rst @@ -4,7 +4,7 @@ Connections =========== Holds settings regarding the different possible connections for search services like Elasticsearch -or Solr. +or Algolia. Configured as:: @@ -22,15 +22,13 @@ Configured as:: Where ``connectionName`` is one of the available :ref:`connections`. -The following settings are available. For each setting its documented which connection consumes it. +The following settings are available. .. _host: ``host`` -------- -Used by: :ref:`Elasticsearch`. - The host, e.g. ``localhost`` or an IP where the search service is reachable from TYPO3 installation. @@ -43,13 +41,8 @@ Example:: ``port`` -------- -Used by: :ref:`Elasticsearch`. - The port where search service is reachable. E.g. default ``9200`` for Elasticsearch. Example:: plugin.tx_searchcore.settings.connections.elasticsearch.port = 9200 - - - diff --git a/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst b/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst index c8653a5..d39b42d 100644 --- a/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst +++ b/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst @@ -1,7 +1,7 @@ ``Codappix\SearchCore\DataProcessing\RemoveProcessor`` ====================================================== -Will remove fields from record, e.g. if you do not want to sent them to elasticsearch at all. +Will remove fields from record. Possible Options: diff --git a/Documentation/source/configuration/dataProcessing/availableAndPlanned.rst b/Documentation/source/configuration/dataProcessing/availableAndPlanned.rst deleted file mode 100644 index 9f31736..0000000 --- a/Documentation/source/configuration/dataProcessing/availableAndPlanned.rst +++ /dev/null @@ -1,36 +0,0 @@ -The following Processor are available: - -.. toctree:: - :maxdepth: 1 - :glob: - - /configuration/dataProcessing/ContentObjectDataProcessorAdapterProcessor - /configuration/dataProcessing/CopyToProcessor - /configuration/dataProcessing/GeoPointProcessor - /configuration/dataProcessing/RemoveProcessor - -The following Processor are planned: - - ``Codappix\SearchCore\DataProcessing\ReplaceProcessor`` - Will execute a search and replace on configured fields. - - ``Codappix\SearchCore\DataProcessing\RootLevelProcessor`` - Will attach the root level to the record. - - ``Codappix\SearchCore\DataProcessing\ChannelProcessor`` - Will add a configurable channel to the record, e.g. if you have different areas in your - website like "products" and "infos". - - ``Codappix\SearchCore\DataProcessing\RelationResolverProcessor`` - Resolves all relations using the TCA. - -Of course you are able to provide further processors. Just implement -``Codappix\SearchCore\DataProcessing\ProcessorInterface`` and use the FQCN (=Fully qualified -class name) as done in the examples above. - -By implementing also the same interface as necessary for TYPO3 -:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`, you are able to reuse the same code -also for Fluid to prepare the same record fetched from DB for your fluid. - -Dependency injection is possible inside of processors, as we instantiate through extbase -``ObjectManager``. diff --git a/Documentation/source/configuration/indexing.rst b/Documentation/source/configuration/indexing.rst index 7d6c3d4..050f0ab 100644 --- a/Documentation/source/configuration/indexing.rst +++ b/Documentation/source/configuration/indexing.rst @@ -40,11 +40,7 @@ Contains a comma separated list of page uids. Spaces are trimmed. Example:: - plugin.tx_searchcore.settings.indexing..rootLineBlacklist = 3, 10, 100 - -Also it's possible to define some behaviour for the different document types. In context of TYPO3 -tables are used as document types 1:1. It's possible to configure different tables. The following -options are available: + plugin.tx_searchcore.settings.indexing.pages.rootLineBlacklist = 3, 10, 100 .. _additionalWhereClause: @@ -55,16 +51,16 @@ Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. Add additional SQL to where clauses to determine indexable records from the table. This way you can exclude specific records like ``tt_content`` records with specific ``CType`` values or -something else. E.g. you can add a new field to the table to exclude records from indexing. +something else. Example:: - plugin.tx_searchcore.settings.indexing..additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu') + plugin.tx_searchcore.settings.indexing.tt_content.additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu') .. attention:: Make sure to prefix all fields with the corresponding table name. The selection from - database will contain joins and can lead to SQL errors if a field exists in multiple tables. + database might contain joins and can lead to SQL errors if a field exists in multiple tables. .. _abstractFields: @@ -73,6 +69,10 @@ abstractFields Used by: :ref:`PagesIndexer`. +.. note:: + + Will be migrated to :ref:`dataprocessors` in the future. + Define which field should be used to provide the auto generated field "search_abstract". The fields have to exist in the record to be indexed. Therefore fields like ``content`` are also possible. @@ -80,7 +80,7 @@ possible. Example:: # As last fallback we use the content of the page - plugin.tx_searchcore.settings.indexing..abstractFields := addToList(content) + plugin.tx_searchcore.settings.indexing.pages.abstractFields := addToList(content) Default:: @@ -91,10 +91,10 @@ Default:: mapping ------- -Used by: Elasticsearch connection while indexing. +Used by: :ref:`connection_elasticsearch` connection while indexing. Define mapping for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/mapping.html -You are able to define the mapping for each property / columns. +You are able to define the mapping for each property / column. Example:: @@ -112,7 +112,7 @@ makes building a facet possible. index ----- -Used by: Elasticsearch connection while indexing. +Used by: :ref:`connection_elasticsearch` connection while indexing. Define index for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/indices-create-index.html @@ -141,36 +141,12 @@ Example:: ``char_filter`` and ``filter`` are a comma separated list of options. -.. _dataProcessing: +.. _indexing_dataProcessing: dataProcessing -------------- Used by: All connections while indexing, due to implementation inside ``AbstractIndexer``. -Configure modifications on each document before sending it to the configured connection. Same as -provided by TYPO3 for :ref:`t3tsref:cobj-fluidtemplate` through -:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`. - -All processors are applied in configured order. Allowing to work with already processed data. - -Example:: - - plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing { - 1 = Codappix\SearchCore\DataProcessing\CopyToProcessor - 1 { - to = search_spellcheck - } - - 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor - 2 { - to = search_all - } - } - -The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards -all fields, including ``search_spellcheck`` will be copied to ``search_all``. - -.. include:: /configuration/dataProcessing/availableAndPlanned.rst - -Also data processors are available for search results too, see :ref:`searching_dataProcessing`. +Configure modifications on each document before sending it to the configured connection. +For full documentation check out :ref:`dataprocessors`. diff --git a/Documentation/source/configuration/searching.rst b/Documentation/source/configuration/searching.rst index 857aa74..52abc07 100644 --- a/Documentation/source/configuration/searching.rst +++ b/Documentation/source/configuration/searching.rst @@ -8,45 +8,50 @@ Searching size ---- -Used by: Elasticsearch connection while building search query. - Defined how many search results should be fetched to be available in search result. Example:: plugin.tx_searchcore.settings.searching.size = 50 -Default if not configured is 10. +Default is ``10``. .. _facets: facets ------ -Used by: Elasticsearch connection while building search query. - Define aggregations for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-aggregations-bucket-terms-aggregation.html -Currently only the term facet is provided. Example:: - plugin.tx_searchcore.settings.searching.facets { - contentTypes { - field = CType + category { + terms { + field = categories } } -The above example will provide a facet with options for all found ``CType`` results together -with a count. + month { + date_histogram { + field = released + interval = month + format = Y-MM-01 + order { + _time = desc + } + } + } + + +The above example will provide a facet with options for all found ``categories`` results together +with a count. Also a facet for ``released`` will be provided. .. _filter: filter ------ -Used by: While building search request. - -Define filter that should be set for all requests. +Define filter that should be set for all search requests. Example:: @@ -54,6 +59,9 @@ Example:: property = value } +Also see :ref:`mapping.filter` to map incoming request information, e.g. from a ``select``, to build +more complex filters. + For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``keyword``. .. _minimumShouldMatch: @@ -61,9 +69,8 @@ For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``key minimumShouldMatch ------------------ -Used by: Elasticsearch connection while building search query. - -Define the minimum match for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-minimum-should-match.html +Define the minimum match for Elasticsearch, have a look at the official docs: +https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-minimum-should-match.html Example:: @@ -74,8 +81,6 @@ Example:: boost ----- -Used by: Elasticsearch connection while building search query. - Define fields that should boost the score for results. Example:: @@ -93,10 +98,9 @@ https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_boosting_query_clauses. fieldValueFactor ---------------- -Used by: Elasticsearch connection while building search query. - -Define a field to use as a factor for scoring. The configuration is passed through to elastic -search ``field_value_factor``, see: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor +Define a field to use as a factor for scoring. The configuration is passed through to Elasticsearch +``field_value_factor``, see: +https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor Example:: @@ -119,34 +123,34 @@ E.g. you submit a filter in form of: .. code-block:: html - - Due to TYPO3 7.x fluid limitations, we build this input ourself. - No longer necessary in 8 and above - - - + + Due to TYPO3 7.x fluid limitations, we build this input ourself. + No longer necessary in 8 and above + + + -This will create a ``distance`` filter with subproperties. To make this filter actually work, you +This will create a ``month`` filter with sub properties. To make this filter actually work, you can add the following TypoScript, which will be added to the filter:: mapping { @@ -167,38 +171,36 @@ can add the following TypoScript, which will be added to the filter:: } ``fields`` has a special meaning here. This will actually map the properties of the filter to fields -in elasticsearch. In above example they do match, but you can also use different names in your form. -On the left hand side is the elasticsearch field name, on the right side the one submitted as a -filter. +in Elasticsearch. On the left hand side is the Elasticsearch field name, on the right side the one +submitted as a filter. -The ``field``, in above example ``released``, will be used as the elasticsearch field for -filtering. This way you can use arbitrary filter names and map them to existing elasticsearch fields. +The ``field``, in above example ``released``, will be used as the Elasticsearch field for +filtering. This way you can use arbitrary filter names and map them to existing Elasticsearch fields. Everything that is configured inside ``raw`` is passed, as is, to search service, e.g. -elasticsearch. +Elasticsearch. .. _fields: fields ------ -Defines the fields to fetch and search from elasticsearch. With the following sub keys: +Defines the fields to fetch and search from Elasticsearch. With the following sub keys: -``query`` defines the fields to search in. Default is ``_all`` from 5.x times of elasticsearch. -Configure a comma separated list of fields to search in. This is necessary if you have configured -special mapping for some fields, or just want to search some fields. -The most hits get ranked highest. The following is an example configuration:: +``query`` defines the fields to search in. Configure a comma separated list of fields to search in. +This is necessary if you have configured special mapping for some fields, or just want to search +some fields. The following is an example configuration:: fields { query = _all, city } -The following sub properties configure the fields to fetch from elasticsearch: +The following sub properties configure the fields to fetch from Elasticsearch: First ``stored_fields`` which is a list of comma separated fields which actually exist and will be added. Typically you will use ``_source`` to fetch the whole indexed fields. -Second is ``script_fields``, which allow you to configure scripted fields for elasticsearch. +Second is ``script_fields``, which allow you to configure scripted fields for Elasticsearch. An example might look like the following:: fields { @@ -221,7 +223,7 @@ In above example we add a single ``script_field`` called ``distance``. We add a field should be added. The condition will be parsed as Fluidtemplate and is casted to bool via PHP. If the condition is true, or no ``condition`` exists, the ``script_field`` will be added to the query. The ``condition`` will be removed and everything else is submitted one to one to -elasticsearch, except each property is run through Fluidtemplate, to allow you to use information +Elasticsearch, except each property is run through Fluidtemplate, to allow you to use information from search request, e.g. to insert latitude and longitude from a filter, like in the above example. .. _sort: @@ -246,38 +248,14 @@ Example:: mode = filter } -Only ``filter`` is allowed as value. Will submit an empty query to switch to filter mode. +Only ``filter`` is allowed as value, as ``search`` is default behaviour. Using ``filter`` will +trigger a search to provide data while visiting the page, possible :ref:`filter` allow you to build +pages like "News". -.. _searching_dataProcessing: +.. _searching_dataprocessing: dataProcessing -------------- -Used by: All connections while indexing, due to implementation inside ``SearchService``. - -Configure modifications on each document before returning search result. Same as provided by TYPO3 -for :ref:`t3tsref:cobj-fluidtemplate` through -:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`. - -All processors are applied in configured order. Allowing to work with already processed data. - -Example:: - - plugin.tx_searchcore.settings.searching.dataProcessing { - 1 = Codappix\SearchCore\DataProcessing\CopyToProcessor - 1 { - to = search_spellcheck - } - - 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor - 2 { - to = search_all - } - } - -The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards -all fields, including ``search_spellcheck`` will be copied to ``search_all``. - -.. include:: /configuration/dataProcessing/availableAndPlanned.rst - -Also data processors are available while indexing too, see :ref:`dataProcessing`. +Configure modifications on each document before returning search result. +For full documentation check out :ref:`dataprocessors`. diff --git a/Documentation/source/connections.rst b/Documentation/source/connections.rst index b51f280..73ce273 100644 --- a/Documentation/source/connections.rst +++ b/Documentation/source/connections.rst @@ -5,26 +5,25 @@ Connections See Concept of :ref:`concepts_connections` for further background information. -The extension provides the following connections out of the box: +For information about implementing a new connection, take a look at :ref:`development_connection`. -.. _Elasticsearch: +The following connections were developed, or are in development, for ``search_core``: + +.. _connection_elasticsearch: Elasticsearch ------------- -Integrates `elastic Elasticsearch`_ using `elastica`_ into TYPO3. +Support for `Elasticsearch`_ is provided out of the box by `search_core` at the moment. -Provides basic support like indexing without mappings and full text search at the moment. +.. _Elasticsearch: https://www.elastic.co/products/elasticsearch -The connection is configurable through the following options: +.. _connection_algolia: -* :ref:`host` +Algolia +------- -* :ref:`port` +`search_algolia`_ will integrate `Algolia`_ and is currently under development by Martin Hummer. -* :ref:`mapping` - -* :ref:`facets` - -.. _elastic Elasticsearch: https://www.elastic.co/products/elasticsearch -.. _elastica: http://elastica.io/ +.. _search_algolia: https://github.com/martinhummer/search_algolia/ +.. _Algolia: https://www.algolia.com/ diff --git a/Documentation/source/dataprocessors.rst b/Documentation/source/dataprocessors.rst new file mode 100644 index 0000000..67ff655 --- /dev/null +++ b/Documentation/source/dataprocessors.rst @@ -0,0 +1,98 @@ +.. _dataprocessors: + +DataProcessors +============== + +See Concept of :ref:`concepts_dataprocessing` for further background information. + +For information about implementing a new DataProcessor, take a look at +:ref:`development_dataprocessor`. + +Same as provided by TYPO3 for :ref:`t3tsref:cobj-fluidtemplate` through +:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`. + +.. _dataprocessors_usage: + +Usage +----- + +All processors are applied in configured order. Allowing to work with already processed data. +They can be applied during indexing and for search results. + +Example for indexing:: + + plugin.tx_searchcore.settings.indexing.pages.dataProcessing { + 1 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 1 { + to = search_spellcheck + } + + 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 2 { + to = search_all + } + } + +The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards +all fields, including ``search_spellcheck`` will be copied to ``search_all``. + +Example for search results:: + + plugin.tx_searchcore.settings.searching.dataProcessing { + 1 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 1 { + to = search_spellcheck + } + + 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 2 { + to = search_all + } + } + +The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards +all fields, including ``search_spellcheck`` will be copied to ``search_all``. + +.. _dataprocessors_availableDataProcessors: + +Available DataProcessors +------------------------ + +.. toctree:: + :maxdepth: 1 + :glob: + + /configuration/dataProcessing/ContentObjectDataProcessorAdapterProcessor + /configuration/dataProcessing/CopyToProcessor + /configuration/dataProcessing/GeoPointProcessor + /configuration/dataProcessing/RemoveProcessor + +.. _dataprocessors_plannedDataProcessors: + +Planned DataProcessors +---------------------- + + ``Codappix\SearchCore\DataProcessing\ReplaceProcessor`` + Will execute a search and replace on configured fields. + + ``Codappix\SearchCore\DataProcessing\RootLevelProcessor`` + Will attach the root level to the record. + + ``Codappix\SearchCore\DataProcessing\ChannelProcessor`` + Will add a configurable channel to the record, e.g. if you have different areas in your + website like "products" and "infos". + + ``Codappix\SearchCore\DataProcessing\RelationResolverProcessor`` + Resolves all relations using the TCA. + This is currently done through indexer. + +.. Of course you are able to provide further processors. Just implement +.. ``Codappix\SearchCore\DataProcessing\ProcessorInterface`` and use the FQCN (=Fully qualified +.. class name) as done in the examples above. + +.. By implementing also the same interface as necessary for TYPO3 +.. :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`, you are able to reuse the same code +.. also for Fluid to prepare the same record fetched from DB for your fluid. + +.. Dependency injection is possible inside of processors, as we instantiate through extbase +.. ``ObjectManager``. diff --git a/Documentation/source/development.rst b/Documentation/source/development.rst index e73425c..e65dcdf 100644 --- a/Documentation/source/development.rst +++ b/Documentation/source/development.rst @@ -1,71 +1,16 @@ .. highlight:: bash -.. _contribution: - -Contribution -============ - -Everyone is welcome to contribute, whether it's code, issues, feature requests or any other kind. - -Below is a documentation what to respect during contributions. - -.. _contribution_setup: - -Setup ------ - -To start contributions regarding code, make sure your environment matches the following -requirements: - -* composer is executable - -* PHP on CLI is executable - -* MySQL is up and running with user *dev* and password *dev* on *127.0.0.1* or to overwrite the - environment variables, see :file:`Makefile`. - And MySQL is not set to strict mode as TYPO3 doesn't support strict mode, see - https://review.typo3.org/#/c/26725/3/INSTALL.md. - -* Elasticsearch is installed and up and running on *localhost:9200*. - -Then setup your system:: - - git clone git@github.com:DanielSiepmann/search_core.git \ - && cd search_core \ - && export typo3DatabaseName="searchcoretest76" \ - && export TYPO3_VERSION="~7.6" \ - && make install \ - && make unitTests \ - && make functionalTests - -If all tests are okay, start your work. - -If you are working with multiple TYPO3 versions make sure to export `typo3DatabaseName` and -`TYPO3_VERSION` in your environment like:: - - export typo3DatabaseName="searchcoretest62" - export TYPO3_VERSION="~6.2" - -Also run the install command for each version before running any tests. Only this will make sure you -are testing against the actual TYPO3 Version and database scheme. - -.. _contribution_development: - Development ------------ +=========== -All changes are introduced through pull requests at `Github`_ and should contain the following: +There are some ways we will cover here. One is how you can develop own parts like Indexer, +DataProcessor and Connection. The other is how to contribute. -* Adjusted tests if tests existed before. Otherwise they will break on `travis-ci`_. +.. toctree:: + :maxdepth: 1 + :glob: -* New tests whenever possible and useful. - -* Code has to follow `PSR-2`_. - -* Adjusted documentation. - -* Make sure to follow the documented :ref:`concepts`. - -.. _Github: https://github.com/DanielSiepmann/search_core -.. _travis-ci: https://travis-ci.org/ -.. _PSR-2: http://www.php-fig.org/psr/psr-2/ + development/indexer + development/dataProcessor + development/connection + development/contribution diff --git a/Documentation/source/development/connection.rst b/Documentation/source/development/connection.rst new file mode 100644 index 0000000..af0ab14 --- /dev/null +++ b/Documentation/source/development/connection.rst @@ -0,0 +1,11 @@ +.. _development_connection: + +Develop a new Connection +======================== + +Make sure you understood :ref:`concepts_connections`. + +Each Connection has to be a single class which implements +``Codappix\SearchCore\Connection\ConnectionInterface``. + +Dependency Injection is working for custom Connections. diff --git a/Documentation/source/development/contribution.rst b/Documentation/source/development/contribution.rst new file mode 100644 index 0000000..32b6400 --- /dev/null +++ b/Documentation/source/development/contribution.rst @@ -0,0 +1,69 @@ +.. _contribution: + +Contribution +============ + +Everyone is welcome to contribute, whether it's code, issues, feature requests or any other kind. + +Below is a documentation what to respect during contributions. + +.. _contribution_setup: + +Setup +----- + +To start contributions regarding code, make sure your environment matches the following +requirements: + +* composer is executable + +* PHP on CLI is executable + +* MySQL is up and running with user *dev* and password *dev* on *127.0.0.1* or to overwrite the + environment variables, see :file:`Makefile`. + And MySQL is not set to strict mode as TYPO3 doesn't support strict mode, see + https://review.typo3.org/#/c/26725/3/INSTALL.md. + +* Elasticsearch is installed and up and running on *localhost:9200*. + +Then setup your system:: + + git clone git@github.com:DanielSiepmann/search_core.git \ + && cd search_core \ + && export typo3DatabaseName="searchcoretest76" \ + && export TYPO3_VERSION="~8.7" \ + && make install \ + && make unitTests \ + && make functionalTests + +If all tests are okay, start your work. + +If you are working with multiple TYPO3 versions make sure to export `typo3DatabaseName` and +`TYPO3_VERSION` in your environment like:: + + export typo3DatabaseName="searchcoretest62" + export TYPO3_VERSION="~6.2" + +Also run the install command for each version before running any tests. Only this will make sure you +are testing against the actual TYPO3 Version and database scheme. + +.. _contribution_development: + +Development +----------- + +All changes are introduced through pull requests at `Github`_ and should contain the following: + +* Adjusted tests if tests existed before. Otherwise they will break on `travis-ci`_. + +* New tests whenever possible and useful. + +* Code has to follow `PSR-2`_. + +* Adjusted documentation. + +* Make sure to follow the documented :ref:`concepts`. + +.. _Github: https://github.com/DanielSiepmann/search_core +.. _travis-ci: https://travis-ci.org/ +.. _PSR-2: http://www.php-fig.org/psr/psr-2/ diff --git a/Documentation/source/development/dataProcessor.rst b/Documentation/source/development/dataProcessor.rst new file mode 100644 index 0000000..af2732b --- /dev/null +++ b/Documentation/source/development/dataProcessor.rst @@ -0,0 +1,15 @@ +.. _development_dataprocessor: + +Develop a new DataProcessor +=========================== + +Make sure you understood :ref:`concepts_dataprocessing`. + +Each DataProcessor has to be a single class which implements +``Codappix\SearchCore\DataProcessing\ProcessorInterface``. + +Make sure you support both, Frontend and Backend, as processors can be called during searching and +indexing. Therefore do not rely on e.g. ``TSFE``, make sure dependencies are met and your code will +work in both environments. + +Dependency Injection is working for custom DataProcessors. diff --git a/Documentation/source/development/indexer.rst b/Documentation/source/development/indexer.rst new file mode 100644 index 0000000..c44aabd --- /dev/null +++ b/Documentation/source/development/indexer.rst @@ -0,0 +1,21 @@ +.. _development_indexer: + +Develop a new Indexer +===================== + +Make sure you understood :ref:`concepts_indexing`. + +Each indexer has to be a single class which implements +``Codappix\SearchCore\Domain\Index\IndexerInterface``. + +The indexer should call the connection with all necessary information about the document(s) to +trigger indexing or deletion of whole index. + +As this is the "indexer", deletion of single documents is directly processed by the connection. + +``setIdentifier`` is called with the identifier of the current Indexer. This might be usefull to +fetch configuration, related to the indexing, from +``Codappix\SearchCore\Configuration\ConfigurationContainerInterface``. + +Dependency Injection is working for custom indexers, therefore you are able to inject the +``ConfigurationContainerInterface``. diff --git a/Documentation/source/features.rst b/Documentation/source/features.rst index 1a4abd6..57014bd 100644 --- a/Documentation/source/features.rst +++ b/Documentation/source/features.rst @@ -3,28 +3,34 @@ Features ======== -The following features are currently provided: +The following features are available: .. _features_indexing: Indexing -------- -Indexing data to Elasticsearch is provided. The extension delivers an indexer for TCA with zero -configuration needs. Still it's possible to configure the indexer. +Indexing of data is possible. We deliver an indexer for TCA with zero configuration needs. You can +also provide custom indexer for arbitrary data which is not indexable through TCA. -Also custom classes can be used as indexers. +Also a finisher for TYPO3 Form-Extension is provided to integrate indexing after something was +update through the Form-Extension. -Furthermore a finisher for TYPO3 Form-Extension is provided to integrate indexing. +Indexing is done through Hooks and CLI. We therefore provide commands to index and delete indexed +data. .. _features_search: Searching --------- -Currently all fields are searched for a single search input. +.. note:: + Currently only integrated for Elasticsearch with no abstraction. + If you need to implement your own search, please open an issue on Github and we will change the code + base. -Also multiple filter are supported. Filtering results by fields for string contents. +Via TypoScript it's possible to configure the fields to query, minimum match and script fields. +Also multiple filter are supported, filtering results by fields. Facets / aggregates are also possible. Therefore a mapping has to be defined in TypoScript for indexing, and the facets itself while searching. @@ -32,19 +38,25 @@ indexing, and the facets itself while searching. .. _features_dataProcessing: DataProcessing -============== +-------------- -DataProcessing, as known from ``FLUIDTEMPLATE`` is available while indexing and for search results. -Each item can be processed by multiple processor to prepare data for indexing and output. +DataProcessing, as known from ``FLUIDTEMPLATE``, is available while indexing and for search results. +Each record and result item can be processed by multiple processor to prepare data for indexing and +output. -See :ref:`concepts_indexing_dataprocessing` in :ref:`concepts` section. +See :ref:`concepts_dataprocessing` in :ref:`concepts` section. .. _features_planned: Planned ---------- +------- The following features are currently planned and will be integrated: -#. Pagination - Add a pagination to search results, to allow users to walk through all results. +#. :issue:`25` Multi language. +#. :issue:`94` Respect access rights while indexing relations. +#. :issue:`75` Configuration of index name (for Elasticsearch). + +For a full list, check out our `open issues`_. + +.. _open issues: https://github.com/Codappix/search_core/issues diff --git a/Documentation/source/index.rst b/Documentation/source/index.rst index 2edfae3..1cc9eb6 100644 --- a/Documentation/source/index.rst +++ b/Documentation/source/index.rst @@ -1,7 +1,7 @@ .. include:: readme.rst Table of Contents -================= +----------------- .. toctree:: :maxdepth: 1 @@ -14,5 +14,6 @@ Table of Contents concepts connections indexer + dataprocessors development changelog diff --git a/Documentation/source/indexer.rst b/Documentation/source/indexer.rst index 3bb10bc..733f5b9 100644 --- a/Documentation/source/indexer.rst +++ b/Documentation/source/indexer.rst @@ -5,6 +5,8 @@ Indexer See Concept of :ref:`concepts_indexing` for further background information. +For information about implementing a new indexer, take a look at :ref:`development_indexer`. + The extension provides the following indexer out of the box: .. _TcaIndexer: @@ -15,23 +17,12 @@ TcaIndexer Provides zero configuration TYPO3 integration by using the :ref:`t3tcaref:start`. You just can start indexing TYPO3. -The indexer will use the TCA to fetch all necessary information like relations. Currently the -implementation is very basic. In future it will also provide mapping for :ref:`Elasticsearch` and -further stuff. +Just add the indexer for a TYPO3 table. The indexer will use the TCA to fetch all necessary +information like relations. -The indexer is configurable through the following options: +.. note:: -* :ref:`rootLineBlacklist` - -* :ref:`additionalWhereClause` - -* :ref:`abstractFields` - -* :ref:`mapping` - -* :ref:`index` - -* :ref:`dataProcessing` + Not all relations are resolved yet, see :issue:`17`. .. _PagesIndexer: @@ -42,23 +33,7 @@ Provides zero configuration TYPO3 integration by using the :ref:`t3tcaref:start` start indexing TYPO3. The indexer will use the TCA to fetch all necessary information like relations. Currently the -implementation is very basic. In future it will also provide mapping for :ref:`Elasticsearch` and -further stuff. Also all static content from each page will be concatenated into a single field to -improve search. - -The indexer is configurable through the following options: - -* :ref:`rootLineBlacklist` - -* :ref:`additionalWhereClause` - -* :ref:`abstractFields` - -* :ref:`mapping` - -* :ref:`index` - -* :ref:`dataProcessing` +implementation is very basic. .. note:: diff --git a/Documentation/source/installation.rst b/Documentation/source/installation.rst index 5bb0c56..0a0e5b6 100644 --- a/Documentation/source/installation.rst +++ b/Documentation/source/installation.rst @@ -4,23 +4,41 @@ Installation ============ +Composer +-------- + The extension can be installed through composer:: - composer require "leonmrni/search_core dev-master as 1.0.x-dev" + composer require "codappix/search_core" "~1.0.0" -or by `downloading`_ and placing it inside the :file:`typo3conf/ext`-Folder of your installation. -In that case you need to install all dependencies yourself. Dependencies are: +Note that you have to allow unstable packages: + +.. code-block:: json + + { + "minimum-stability": "dev", + "prefer-stable": true + } + +Download +-------- + +You can also `download`_ the extension and placing it inside the :file:`typo3conf/ext`-Folder of +your installation. In that case you need to install all dependencies yourself. Dependencies are: .. literalinclude:: ../../composer.json :caption: Dependencies from composer.json :lines: 19-21 :dedent: 8 +Setup +----- + Afterwards you need to enable the extension through the extension manager and include the static TypoScript setup. -If you **don't** want to use the included elasticsearch integration, you have to disable it in the +If you **don't** want to use the included Elasticsearch integration, you have to disable it in the extension manager configuration of the extension by checking the checkbox. It's currently enabled by default but will be moved into its own extension in the future. -.. _downloading: https://github.com/DanielSiepmann/search_core/archive/master.zip +.. _download: https://github.com/codappix/search_core/archive/develop.zip diff --git a/Documentation/source/readme.rst b/Documentation/source/readme.rst index ba77ae8..f4ad917 100644 --- a/Documentation/source/readme.rst +++ b/Documentation/source/readme.rst @@ -1,5 +1,5 @@ -TYPO3 Extension search_core's documentation! -============================================ +TYPO3 Extension search_core +=========================== Introduction ============ @@ -8,24 +8,21 @@ What does it do? ---------------- The goal of this extension is to provide search integrations into TYPO3 CMS. The extension will -abstract the concrete implementations to allow exchange of concrete backends like Elasticsearch or -solr. +provide a convenient API to allow developers to provide concrete implementations of backends like +Elasticsearch, Algolia or Solr. The extension provides integration into TYPO3 like a frontend plugin for searches and hooks to update search indexes on updates. Also a command line interface is provided for interactions like -reindexing. +re-indexing. Current state ------------- This is still a very early beta version. More information can be taken from Github at -`current issues`_ and `current projects`_. +`current issues`_. -We are also focusing on Code Quality and Testing through `travis ci`_, `scrutinizer`_ and `codacy`_. +We are also focusing on Code Quality and Testing through `travis ci`_, ``phpcs``, ``phpunit`` and +``phpstan``. .. _current issues: https://github.com/Codappix/search_core/issues -.. _current projects: https://github.com/Codappix/search_core/projects .. _travis ci: https://travis-ci.org/Codappix/search_core -.. _scrutinizer: https://scrutinizer-ci.com/g/Codappix/search_core/inspections -.. _codacy: https://www.codacy.com/app/Codappix/search_core/dashboard - diff --git a/Documentation/source/usage.rst b/Documentation/source/usage.rst index d57d643..fc6d08a 100644 --- a/Documentation/source/usage.rst +++ b/Documentation/source/usage.rst @@ -11,9 +11,10 @@ Manual indexing You can trigger indexing from CLI:: - ./typo3/cli_dispatch.phpsh extbase index:index --identifier 'tt_content' + ./typo3/cli_dispatch.phpsh extbase index:index --identifier 'pages' + ./bin/typo3cms index:index --identifier 'pages' -This will index the table ``tt_content`` using the :ref:`TcaIndexer`. +This will index the table ``pages`` using the :ref:`TcaIndexer`. Only one index per call is available, to run multiple indexers, just make multiple calls. The indexers have to be defined in TypoScript via :ref:`configuration_options_index`. @@ -25,9 +26,10 @@ Manual deletion You can trigger deletion for a single index from CLI:: - ./typo3/cli_dispatch.phpsh extbase index:delete --identifier 'tt_content' + ./typo3/cli_dispatch.phpsh extbase index:delete --identifier 'pages' + ./bin/typo3cms index:delete --identifier 'pages' -This will delete the index for the table ``tt_content``. +This will delete the index for the table ``pages``. Only one delete per call is available, to run multiple deletions, just make multiple calls. @@ -53,7 +55,6 @@ A form finisher is provided to integrate indexing into form extension. Add form finisher to your available finishers and configure it like: .. code-block:: yaml - :linenos: - identifier: SearchCoreIndexer @@ -62,7 +63,7 @@ Add form finisher to your available finishers and configure it like: indexIdentifier: 'fe_users' recordUid: '{FeUser.user.uid}' -All three options are necessary, where +All three options are necessary, where: ``action`` Is one of ``delete``, ``update`` or ``add``. @@ -81,7 +82,7 @@ plugin. The plugin is named *Search Core*. Please provide your own template, the extension will not deliver a useful template for now. -The extbase mapping is used, this way you can create a form: +The Extbase mapping is used, this way you can create a form: .. code-block:: html @@ -95,11 +96,10 @@ The extbase mapping is used, this way you can create a form: Filter """""" -Thanks to extbase mapping, filter are added to the form: +Thanks to Extbase mapping, filter are added to the form: .. code-block:: html - .. _usage_searching_facets: diff --git a/composer.json b/composer.json index a630385..a935819 100644 --- a/composer.json +++ b/composer.json @@ -37,6 +37,9 @@ ] }, "extra": { + "branch-alias": { + "dev-develop": "0.0.1" + }, "typo3/cms": { "cms-package-dir": "{$vendor-dir}/typo3/cms", "web-dir": ".Build/web" From 3bebfac7698541f0d9da6775e8f648b5683f418b Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 8 Mar 2018 19:26:36 +0100 Subject: [PATCH 34/75] TASK: Fix documentation As we have moved the repository at github, we should update all mentions to use new url. Also a small typo was fixed. --- Documentation/source/development/contribution.rst | 4 ++-- Documentation/source/development/indexer.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation/source/development/contribution.rst b/Documentation/source/development/contribution.rst index 32b6400..c362c37 100644 --- a/Documentation/source/development/contribution.rst +++ b/Documentation/source/development/contribution.rst @@ -28,7 +28,7 @@ requirements: Then setup your system:: - git clone git@github.com:DanielSiepmann/search_core.git \ + git clone git@github.com:codappix/search_core.git \ && cd search_core \ && export typo3DatabaseName="searchcoretest76" \ && export TYPO3_VERSION="~8.7" \ @@ -64,6 +64,6 @@ All changes are introduced through pull requests at `Github`_ and should contain * Make sure to follow the documented :ref:`concepts`. -.. _Github: https://github.com/DanielSiepmann/search_core +.. _Github: https://github.com/codappix/search_core .. _travis-ci: https://travis-ci.org/ .. _PSR-2: http://www.php-fig.org/psr/psr-2/ diff --git a/Documentation/source/development/indexer.rst b/Documentation/source/development/indexer.rst index c44aabd..d5a067e 100644 --- a/Documentation/source/development/indexer.rst +++ b/Documentation/source/development/indexer.rst @@ -13,7 +13,7 @@ trigger indexing or deletion of whole index. As this is the "indexer", deletion of single documents is directly processed by the connection. -``setIdentifier`` is called with the identifier of the current Indexer. This might be usefull to +``setIdentifier`` is called with the identifier of the current Indexer. This might be useful to fetch configuration, related to the indexing, from ``Codappix\SearchCore\Configuration\ConfigurationContainerInterface``. From a469f63aa651a2a6294846103740ca4ce1b505c2 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 13 Mar 2018 10:51:59 +0100 Subject: [PATCH 35/75] TASK: Cleanup differences We have some small differences between develop and support/76 branch. This differences are just CGL and should not exist. --- Classes/Domain/Index/AbstractIndexer.php | 4 ---- Classes/Domain/Index/TcaIndexer/RelationResolver.php | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/Classes/Domain/Index/AbstractIndexer.php b/Classes/Domain/Index/AbstractIndexer.php index d264b08..6644716 100644 --- a/Classes/Domain/Index/AbstractIndexer.php +++ b/Classes/Domain/Index/AbstractIndexer.php @@ -68,10 +68,6 @@ abstract class AbstractIndexer implements IndexerInterface $this->identifier = $identifier; } - /** - * @param ConnectionInterface $connection - * @param ConfigurationContainerInterface $configuration - */ public function __construct(ConnectionInterface $connection, ConfigurationContainerInterface $configuration) { $this->connection = $connection; diff --git a/Classes/Domain/Index/TcaIndexer/RelationResolver.php b/Classes/Domain/Index/TcaIndexer/RelationResolver.php index fb19a4f..6af330e 100644 --- a/Classes/Domain/Index/TcaIndexer/RelationResolver.php +++ b/Classes/Domain/Index/TcaIndexer/RelationResolver.php @@ -97,7 +97,7 @@ class RelationResolver implements Singleton return array_map('trim', explode(',', $value)); } - protected function getUtilityForMode(): string + protected function getUtilityForMode() : string { if (TYPO3_MODE === 'BE') { return BackendUtility::class; From ba19537f4e9e10800a41a1a7a12f3243a9a54e92 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 13 Mar 2018 11:02:29 +0100 Subject: [PATCH 36/75] TASK: Migrate existing 7.6 features We had some features in 7.6 support which we didn't merge up yet. Mostly very small bug fixes or more helpful logging and processing of elasticsearch options. But also adding images of content elements while indexing pages. --- .../Connection/Elasticsearch/IndexFactory.php | 15 +++-- .../Domain/Index/TcaIndexer/PagesIndexer.php | 55 ++++++++++++++++--- Classes/Domain/Search/QueryFactory.php | 14 +++-- 3 files changed, 66 insertions(+), 18 deletions(-) diff --git a/Classes/Connection/Elasticsearch/IndexFactory.php b/Classes/Connection/Elasticsearch/IndexFactory.php index 8b57783..c56892a 100644 --- a/Classes/Connection/Elasticsearch/IndexFactory.php +++ b/Classes/Connection/Elasticsearch/IndexFactory.php @@ -53,7 +53,10 @@ class IndexFactory implements Singleton $index = $connection->getClient()->getIndex('typo3content'); if ($index->exists() === false) { - $index->create($this->getConfigurationFor($documentType)); + $config = $this->getConfigurationFor($documentType); + $this->logger->debug(sprintf('Create index %s.', $documentType), [$documentType, $config]); + $index->create($config); + $this->logger->debug(sprintf('Created index %s.', $documentType), [$documentType]); } return $index; @@ -64,9 +67,11 @@ class IndexFactory implements Singleton try { $configuration = $this->configuration->get('indexing.' . $documentType . '.index'); - if (isset($configuration['analysis']['analyzer'])) { - foreach ($configuration['analysis']['analyzer'] as $key => $analyzer) { - $configuration['analysis']['analyzer'][$key] = $this->prepareAnalyzerConfiguration($analyzer); + foreach (['analyzer', 'filter'] as $optionsToExpand) { + if (isset($configuration['analysis'][$optionsToExpand])) { + foreach ($configuration['analysis'][$optionsToExpand] as $key => $options) { + $configuration['analysis'][$optionsToExpand][$key] = $this->prepareOptions($options); + } } } @@ -78,7 +83,7 @@ class IndexFactory implements Singleton protected function prepareAnalyzerConfiguration(array $analyzer) : array { - $fieldsToExplode = ['char_filter', 'filter']; + $fieldsToExplode = ['char_filter', 'filter', 'word_list']; foreach ($fieldsToExplode as $fieldToExplode) { if (isset($analyzer[$fieldToExplode])) { diff --git a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php index 4f6d03d..8882696 100644 --- a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php +++ b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php @@ -34,6 +34,12 @@ class PagesIndexer extends TcaIndexer */ protected $contentTableService; + /** + * @var \TYPO3\CMS\Core\Resource\FileRepository + * @inject + */ + protected $fileRepository; + /** * @param TcaTableService $tcaTableService * @param TcaTableService $contentTableService @@ -60,28 +66,63 @@ class PagesIndexer extends TcaIndexer } } - $record['content'] = $this->fetchContentForPage($record['uid']); + $record['media'] = $this->fetchMediaForPage($record['uid']); + $content = $this->fetchContentForPage($record['uid']); + if ($content !== []) { + $record['content'] = $content['content']; + $record['media'] = array_values(array_unique(array_merge($record['media'], $content['images']))); + } parent::prepareRecord($record); } - protected function fetchContentForPage(int $uid) : string + protected function fetchContentForPage(int $uid) : array { $contentElements = $this->getQuery($this->contentTableService)->execute()->fetchAll(); if ($contentElements === null) { $this->logger->debug('No content for page ' . $uid); - return ''; + return []; } $this->logger->debug('Fetched content for page ' . $uid); + $images = []; $content = []; foreach ($contentElements as $contentElement) { + $images = array_merge( + $images, + $this->getContentElementImages($contentElement['uid']) + ); $content[] = $contentElement['bodytext']; } - // Remove Tags. - // Interpret escaped new lines and special chars. - // Trim, e.g. trailing or leading new lines. - return trim(stripcslashes(strip_tags(implode(' ', $content)))); + return [ + // Remove Tags. + // Interpret escaped new lines and special chars. + // Trim, e.g. trailing or leading new lines. + 'content' => trim(stripcslashes(strip_tags(implode(' ', $content)))), + 'images' => $images, + ]; + } + + protected function getContentElementImages(int $uidOfContentElement) : array + { + return $this->fetchSysFileReferenceUids($uidOfContentElement, 'tt_content', 'image'); + } + + protected function fetchMediaForPage(int $uid) : array + { + return $this->fetchSysFileReferenceUids($uid, 'pages', 'media'); + } + + protected function fetchSysFileReferenceUids(int $uid, string $tablename, string $fieldname) : array + { + $imageRelationUids = []; + $imageRelations = $this->fileRepository->findByRelation($tablename, $fieldname, $uid); + + foreach ($imageRelations as $relation) { + $imageRelationUids[] = $relation->getUid(); + } + + return $imageRelationUids; } } diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 80c5151..98e3324 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -136,13 +136,15 @@ class QueryFactory ]; } - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ - 'query' => [ - 'bool' => [ - 'should' => $boostQueryParts, + if (!empty($boostQueryParts)) { + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ + 'query' => [ + 'bool' => [ + 'should' => $boostQueryParts, + ], ], - ], - ]); + ]); + } } protected function addFactorBoost(array &$query) From 75335e2bc145657fccbbceec267e23bfab0a7c53 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 13 Mar 2018 11:03:57 +0100 Subject: [PATCH 37/75] TASK: Do not import unnecessary namespace As imported class is on same level as we are, we do not need to import the class. --- Classes/Domain/Index/IndexerFactory.php | 1 - 1 file changed, 1 deletion(-) diff --git a/Classes/Domain/Index/IndexerFactory.php b/Classes/Domain/Index/IndexerFactory.php index 6efe035..b1a431d 100644 --- a/Classes/Domain/Index/IndexerFactory.php +++ b/Classes/Domain/Index/IndexerFactory.php @@ -23,7 +23,6 @@ namespace Codappix\SearchCore\Domain\Index; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Domain\Index\IndexerInterface; -use Codappix\SearchCore\Domain\Index\TcaIndexer; use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService; use TYPO3\CMS\Core\SingletonInterface as Singleton; use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; From 16bc22aa44282d7e668476336513139e19a59d6f Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 13 Mar 2018 11:28:40 +0100 Subject: [PATCH 38/75] TASK: Support tests for TYPO3 CMS 7.6 Check which version to run and switch TYPO3 bootstrapping. Also allow tests to run with TYPO3 CMS 7.6 again. --- .travis.yml | 3 +++ .../source/development/contribution.rst | 6 ++--- Makefile | 11 ++++++++- Tests/Functional/Bootstrap.php | 9 +++++++ Tests/Functional/FunctionalTests.xml | 3 +-- Tests/InstallPatches/composer.json.patch | 14 +++++++++++ Tests/Unit/Bootstrap.php | 9 +++++++ Tests/Unit/UnitTests.xml | 2 +- composer.json | 2 +- ext_emconf.php | 6 ++--- ext_localconf.php | 24 +++++++++++++++---- 11 files changed, 73 insertions(+), 16 deletions(-) create mode 100644 Tests/Functional/Bootstrap.php create mode 100644 Tests/InstallPatches/composer.json.patch create mode 100644 Tests/Unit/Bootstrap.php diff --git a/.travis.yml b/.travis.yml index a765673..2babbf3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,7 @@ language: php php: - 7.0 - 7.1 + - 7.2 env: global: @@ -24,6 +25,8 @@ env: - typo3DatabaseHost="127.0.0.1" - typo3DatabaseUsername="travis" - typo3DatabasePassword="" + matrix: + - TYPO3_VERSION="~7.6" matrix: fast_finish: true diff --git a/Documentation/source/development/contribution.rst b/Documentation/source/development/contribution.rst index c362c37..f65bb21 100644 --- a/Documentation/source/development/contribution.rst +++ b/Documentation/source/development/contribution.rst @@ -30,7 +30,7 @@ Then setup your system:: git clone git@github.com:codappix/search_core.git \ && cd search_core \ - && export typo3DatabaseName="searchcoretest76" \ + && export typo3DatabaseName="searchcoretest87" \ && export TYPO3_VERSION="~8.7" \ && make install \ && make unitTests \ @@ -41,8 +41,8 @@ If all tests are okay, start your work. If you are working with multiple TYPO3 versions make sure to export `typo3DatabaseName` and `TYPO3_VERSION` in your environment like:: - export typo3DatabaseName="searchcoretest62" - export TYPO3_VERSION="~6.2" + export typo3DatabaseName="searchcoretest76" + export TYPO3_VERSION="~7.6" Also run the install command for each version before running any tests. Only this will make sure you are testing against the actual TYPO3 Version and database scheme. diff --git a/Makefile b/Makefile index b8f4ac5..ac3d7d5 100644 --- a/Makefile +++ b/Makefile @@ -10,9 +10,18 @@ typo3DatabaseUsername ?= "dev" typo3DatabasePassword ?= "dev" typo3DatabaseHost ?= "127.0.0.1" +sourceOrDist=--prefer-dist +ifeq ($(TYPO3_VERSION),~7.6) + sourceOrDist=--prefer-source +endif + .PHONY: install install: clean - COMPOSER_PROCESS_TIMEOUT=1000 composer require -vv --dev --prefer-dist typo3/cms="$(TYPO3_VERSION)" + if [ $(TYPO3_VERSION) = ~7.6 ]; then \ + patch composer.json Tests/InstallPatches/composer.json.patch; \ + fi + + COMPOSER_PROCESS_TIMEOUT=1000 composer require -vv --dev $(sourceOrDist) typo3/cms="$(TYPO3_VERSION)" git checkout composer.json cgl: diff --git a/Tests/Functional/Bootstrap.php b/Tests/Functional/Bootstrap.php new file mode 100644 index 0000000..66b8eba --- /dev/null +++ b/Tests/Functional/Bootstrap.php @@ -0,0 +1,9 @@ +=7.0.0", - "typo3/cms": "~8.7", + "typo3/cms": ">= 7.6.0 < 9.0.0", "ruflin/elastica": "~3.2" }, "require-dev": { diff --git a/ext_emconf.php b/ext_emconf.php index 4d342a5..7c3a087 100644 --- a/ext_emconf.php +++ b/ext_emconf.php @@ -7,8 +7,8 @@ $EM_CONF[$_EXTKEY] = [ 'clearCacheOnLoad' => 1, 'constraints' => [ 'depends' => [ - 'typo3' => '8.7.0-8.7.99', - 'php' => '7.1.0-7.99.99' + 'typo3' => '7.6.0-8.7.99', + 'php' => '7.0.0-7.2.99' ], 'conflicts' => [], ], @@ -18,7 +18,7 @@ $EM_CONF[$_EXTKEY] = [ ], ], 'state' => 'beta', - 'version' => '1.0.0', + 'version' => '0.0.1', 'author' => 'Daniel Siepmann', 'author_email' => 'coding@daniel-siepmann.de', ]; diff --git a/ext_localconf.php b/ext_localconf.php index a713d0b..d4d7cef 100644 --- a/ext_localconf.php +++ b/ext_localconf.php @@ -37,17 +37,31 @@ call_user_func( ] ); + // Register different concrete implementations, depending on current TYPO3 version. + // This way we can provide working implementations for multiple TYPO3 versions. + $container = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Extbase\Object\Container\Container::class); + if (\TYPO3\CMS\Core\Utility\VersionNumberUtility::convertVersionNumberToInteger(TYPO3_version) >= 8000000) { + $container->registerImplementation( + \Codappix\SearchCore\Compatibility\TypoScriptServiceInterface::class, + \TYPO3\CMS\Core\TypoScript\TypoScriptService::class + ); + } else { + $container->registerImplementation( + \Codappix\SearchCore\Compatibility\TypoScriptServiceInterface::class, + \TYPO3\CMS\Extbase\Service\TypoScriptService::class + ); + } + // API does make use of object manager, therefore use GLOBALS $extensionConfiguration = unserialize($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf'][$extensionKey]); if ($extensionConfiguration === false || !isset($extensionConfiguration['disable.']['elasticsearch']) || $extensionConfiguration['disable.']['elasticsearch'] !== '1' ) { - \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Extbase\Object\Container\Container::class) - ->registerImplementation( - \Codappix\SearchCore\Connection\ConnectionInterface::class, - \Codappix\SearchCore\Connection\Elasticsearch::class - ); + $container->registerImplementation( + \Codappix\SearchCore\Connection\ConnectionInterface::class, + \Codappix\SearchCore\Connection\Elasticsearch::class + ); } }, $_EXTKEY From 9e80574361b2791d53876045bdb3f26d4bfc517f Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 13 Mar 2018 12:18:00 +0100 Subject: [PATCH 39/75] TASK: Provide compatible TypoScriptService for both TYPO3 versions --- Classes/Compatibility/TypoScriptService.php | 31 +++++++++++++++++++ Classes/Compatibility/TypoScriptService76.php | 31 +++++++++++++++++++ .../TypoScriptServiceInterface.php | 30 ++++++++++++++++++ ...entObjectDataProcessorAdapterProcessor.php | 6 ++-- .../Functional/AbstractFunctionalTestCase.php | 5 +++ ...bjectDataProcessorAdapterProcessorTest.php | 11 +++++-- ext_localconf.php | 4 +-- 7 files changed, 111 insertions(+), 7 deletions(-) create mode 100644 Classes/Compatibility/TypoScriptService.php create mode 100644 Classes/Compatibility/TypoScriptService76.php create mode 100644 Classes/Compatibility/TypoScriptServiceInterface.php diff --git a/Classes/Compatibility/TypoScriptService.php b/Classes/Compatibility/TypoScriptService.php new file mode 100644 index 0000000..e5aa788 --- /dev/null +++ b/Classes/Compatibility/TypoScriptService.php @@ -0,0 +1,31 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use TYPO3\CMS\Core\TypoScript\TypoScriptService as CoreTypoScriptService; + +/** + * Used since TYPO3 CMS 8.7. + */ +class TypoScriptService extends CoreTypoScriptService implements TypoScriptServiceInterface +{ + +} diff --git a/Classes/Compatibility/TypoScriptService76.php b/Classes/Compatibility/TypoScriptService76.php new file mode 100644 index 0000000..9df82ea --- /dev/null +++ b/Classes/Compatibility/TypoScriptService76.php @@ -0,0 +1,31 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use TYPO3\CMS\Extbase\Service\TypoScriptService as CoreTypoScriptService; + +/** + * Used before TYPO3 CMS 8.7. + */ +class TypoScriptService76 extends CoreTypoScriptService implements TypoScriptServiceInterface +{ + +} diff --git a/Classes/Compatibility/TypoScriptServiceInterface.php b/Classes/Compatibility/TypoScriptServiceInterface.php new file mode 100644 index 0000000..8e6a9e4 --- /dev/null +++ b/Classes/Compatibility/TypoScriptServiceInterface.php @@ -0,0 +1,30 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * Allows to use DI configuration to switch concrete implementation, depending + * on current TYPO3 Version. + */ +interface TypoScriptServiceInterface +{ + public function convertPlainArrayToTypoScriptArray(array $plainArray); +} diff --git a/Classes/DataProcessing/ContentObjectDataProcessorAdapterProcessor.php b/Classes/DataProcessing/ContentObjectDataProcessorAdapterProcessor.php index c3b4c32..2b7f553 100644 --- a/Classes/DataProcessing/ContentObjectDataProcessorAdapterProcessor.php +++ b/Classes/DataProcessing/ContentObjectDataProcessorAdapterProcessor.php @@ -20,8 +20,8 @@ namespace Codappix\SearchCore\DataProcessing; * 02110-1301, USA. */ +use Codappix\SearchCore\Compatibility\TypoScriptServiceInterface; use TYPO3\CMS\Core\Utility\GeneralUtility; -use TYPO3\CMS\Core\TypoScript\TypoScriptService; use TYPO3\CMS\Frontend\ContentObject\ContentObjectRenderer; /** @@ -30,11 +30,11 @@ use TYPO3\CMS\Frontend\ContentObject\ContentObjectRenderer; class ContentObjectDataProcessorAdapterProcessor implements ProcessorInterface { /** - * @var TypoScriptService + * @var TypoScriptServiceInterface */ protected $typoScriptService; - public function __construct(TypoScriptService $typoScriptService) + public function __construct(TypoScriptServiceInterface $typoScriptService) { $this->typoScriptService = $typoScriptService; } diff --git a/Tests/Functional/AbstractFunctionalTestCase.php b/Tests/Functional/AbstractFunctionalTestCase.php index 7f808c7..1287477 100644 --- a/Tests/Functional/AbstractFunctionalTestCase.php +++ b/Tests/Functional/AbstractFunctionalTestCase.php @@ -66,4 +66,9 @@ abstract class AbstractFunctionalTestCase extends CoreTestCase { return ['EXT:search_core/Tests/Functional/Fixtures/BasicSetup.ts']; } + + protected function useLegacyVersion() : bool + { + return \TYPO3\CMS\Core\Utility\VersionNumberUtility::convertVersionNumberToInteger(TYPO3_version) < 8000000; + } } diff --git a/Tests/Functional/DataProcessing/ContentObjectDataProcessorAdapterProcessorTest.php b/Tests/Functional/DataProcessing/ContentObjectDataProcessorAdapterProcessorTest.php index 871b205..029e51d 100644 --- a/Tests/Functional/DataProcessing/ContentObjectDataProcessorAdapterProcessorTest.php +++ b/Tests/Functional/DataProcessing/ContentObjectDataProcessorAdapterProcessorTest.php @@ -20,9 +20,10 @@ namespace Codappix\SearchCore\Tests\Functional\DataProcessing; * 02110-1301, USA. */ +use Codappix\SearchCore\Compatibility\TypoScriptService76; +use Codappix\SearchCore\Compatibility\TypoScriptService; use Codappix\SearchCore\DataProcessing\ContentObjectDataProcessorAdapterProcessor; use Codappix\SearchCore\Tests\Functional\AbstractFunctionalTestCase; -use TYPO3\CMS\Extbase\Service\TypoScriptService; use TYPO3\CMS\Frontend\DataProcessing\SplitProcessor; class ContentObjectDataProcessorAdapterProcessorTest extends AbstractFunctionalTestCase @@ -44,7 +45,13 @@ class ContentObjectDataProcessorAdapterProcessorTest extends AbstractFunctionalT 'new_content' => ['value1', 'value2'], ]; - $subject = new ContentObjectDataProcessorAdapterProcessor(new TypoScriptService); + if ($this->useLegacyVersion()) { + $typoScriptService = new TypoScriptService76(); + } else { + $typoScriptService = new TypoScriptService(); + } + + $subject = new ContentObjectDataProcessorAdapterProcessor($typoScriptService); $processedData = $subject->processData($record, $configuration); $this->assertSame( $expectedData, diff --git a/ext_localconf.php b/ext_localconf.php index d4d7cef..627f33b 100644 --- a/ext_localconf.php +++ b/ext_localconf.php @@ -43,12 +43,12 @@ call_user_func( if (\TYPO3\CMS\Core\Utility\VersionNumberUtility::convertVersionNumberToInteger(TYPO3_version) >= 8000000) { $container->registerImplementation( \Codappix\SearchCore\Compatibility\TypoScriptServiceInterface::class, - \TYPO3\CMS\Core\TypoScript\TypoScriptService::class + \Codappix\SearchCore\Compatibility\TypoScriptService::class ); } else { $container->registerImplementation( \Codappix\SearchCore\Compatibility\TypoScriptServiceInterface::class, - \TYPO3\CMS\Extbase\Service\TypoScriptService::class + \Codappix\SearchCore\Compatibility\TypoScriptService76::class ); } From 6d7199ccf2ae4504ba0a536c519ba9a3d7293b0d Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 13 Mar 2018 12:19:30 +0100 Subject: [PATCH 40/75] TASK: Provide already used logger As we use the logger, we should inject it. --- Classes/Connection/Elasticsearch/IndexFactory.php | 15 +++++++++++++++ .../Connection/Elasticsearch/IndexFactoryTest.php | 1 + 2 files changed, 16 insertions(+) diff --git a/Classes/Connection/Elasticsearch/IndexFactory.php b/Classes/Connection/Elasticsearch/IndexFactory.php index c56892a..d274db7 100644 --- a/Classes/Connection/Elasticsearch/IndexFactory.php +++ b/Classes/Connection/Elasticsearch/IndexFactory.php @@ -37,6 +37,21 @@ class IndexFactory implements Singleton */ protected $configuration; + /** + * @var \TYPO3\CMS\Core\Log\Logger + */ + protected $logger; + + /** + * Inject log manager to get concrete logger from it. + * + * @param \TYPO3\CMS\Core\Log\LogManager $logManager + */ + public function injectLogger(\TYPO3\CMS\Core\Log\LogManager $logManager) + { + $this->logger = $logManager->getLogger(__CLASS__); + } + /** * @param ConfigurationContainerInterface $configuration */ diff --git a/Tests/Unit/Connection/Elasticsearch/IndexFactoryTest.php b/Tests/Unit/Connection/Elasticsearch/IndexFactoryTest.php index c73fe36..4ede3cb 100644 --- a/Tests/Unit/Connection/Elasticsearch/IndexFactoryTest.php +++ b/Tests/Unit/Connection/Elasticsearch/IndexFactoryTest.php @@ -38,6 +38,7 @@ class IndexFactoryTest extends AbstractUnitTestCase $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock(); $this->subject = new IndexFactory($this->configuration); + $this->subject->injectLogger($this->getMockedLogger()); } /** From 0122dd88e47e0f7272f428936e2228fed5de3531 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 13 Mar 2018 12:19:54 +0100 Subject: [PATCH 41/75] TASK: Fix call to non existing method --- Classes/Connection/Elasticsearch/IndexFactory.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Classes/Connection/Elasticsearch/IndexFactory.php b/Classes/Connection/Elasticsearch/IndexFactory.php index d274db7..6493c3c 100644 --- a/Classes/Connection/Elasticsearch/IndexFactory.php +++ b/Classes/Connection/Elasticsearch/IndexFactory.php @@ -85,7 +85,7 @@ class IndexFactory implements Singleton foreach (['analyzer', 'filter'] as $optionsToExpand) { if (isset($configuration['analysis'][$optionsToExpand])) { foreach ($configuration['analysis'][$optionsToExpand] as $key => $options) { - $configuration['analysis'][$optionsToExpand][$key] = $this->prepareOptions($options); + $configuration['analysis'][$optionsToExpand][$key] = $this->prepareAnalyzerConfiguration($options); } } } From b2a63e9cb028e6ecce4c35de087cca19ba70d61d Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 13 Mar 2018 17:28:50 +0100 Subject: [PATCH 42/75] TASK: Make extension compatible with CMS 7.6 --- .travis.yml | 1 + .../ImplementationRegistrationService.php | 56 +++ .../Connection/Elasticsearch/IndexFactory.php | 4 +- Classes/Domain/Index/IndexerFactory.php | 8 +- Classes/Domain/Index/TcaIndexer.php | 51 +-- .../Domain/Index/TcaIndexer/PagesIndexer.php | 23 +- .../Index/TcaIndexer/RelationResolver.php | 2 +- .../Index/TcaIndexer/TcaTableService.php | 59 ++- .../Index/TcaIndexer/TcaTableService76.php | 379 ++++++++++++++++++ .../TcaIndexer/TcaTableServiceInterface.php | 44 ++ Makefile | 2 +- .../Functional/AbstractFunctionalTestCase.php | 4 +- .../Connection/Elasticsearch/FilterTest.php | 2 +- .../Elasticsearch/IndexTcaTableTest.php | 17 +- ...bjectDataProcessorAdapterProcessorTest.php | 2 +- .../ProcessesAllowedTablesTest.php | 14 + Tests/Functional/Indexing/TcaIndexerTest.php | 4 +- Tests/Unit/AbstractUnitTestCase.php | 5 + .../Index/TcaIndexer/TcaTableServiceTest.php | 62 ++- .../Form/Finisher/DataHandlerFinisherTest.php | 2 + ext_localconf.php | 16 +- 21 files changed, 646 insertions(+), 111 deletions(-) create mode 100644 Classes/Compatibility/ImplementationRegistrationService.php create mode 100644 Classes/Domain/Index/TcaIndexer/TcaTableService76.php create mode 100644 Classes/Domain/Index/TcaIndexer/TcaTableServiceInterface.php diff --git a/.travis.yml b/.travis.yml index 2babbf3..56f42fa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,6 +27,7 @@ env: - typo3DatabasePassword="" matrix: - TYPO3_VERSION="~7.6" + - TYPO3_VERSION="~8.7" matrix: fast_finish: true diff --git a/Classes/Compatibility/ImplementationRegistrationService.php b/Classes/Compatibility/ImplementationRegistrationService.php new file mode 100644 index 0000000..fa3faa8 --- /dev/null +++ b/Classes/Compatibility/ImplementationRegistrationService.php @@ -0,0 +1,56 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use TYPO3\CMS\Core\Utility\GeneralUtility; +use TYPO3\CMS\Core\Utility\VersionNumberUtility; +use TYPO3\CMS\Extbase\Object\Container\Container; + +/** + * Register different concrete implementations, depending on current TYPO3 version. + * This way we can provide working implementations for multiple TYPO3 versions. + */ +class ImplementationRegistrationService +{ + public static function registerImplementations() + { + $container = GeneralUtility::makeInstance(Container::class); + if (VersionNumberUtility::convertVersionNumberToInteger(TYPO3_version) >= 8000000) { + $container->registerImplementation( + \Codappix\SearchCore\Compatibility\TypoScriptServiceInterface::class, + \Codappix\SearchCore\Compatibility\TypoScriptService::class + ); + $container->registerImplementation( + \Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface::class, + \Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService::class + ); + } else { + $container->registerImplementation( + \Codappix\SearchCore\Compatibility\TypoScriptServiceInterface::class, + \Codappix\SearchCore\Compatibility\TypoScriptService76::class + ); + $container->registerImplementation( + \Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface::class, + \Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService76::class + ); + } + } +} diff --git a/Classes/Connection/Elasticsearch/IndexFactory.php b/Classes/Connection/Elasticsearch/IndexFactory.php index 6493c3c..2ef905f 100644 --- a/Classes/Connection/Elasticsearch/IndexFactory.php +++ b/Classes/Connection/Elasticsearch/IndexFactory.php @@ -85,7 +85,9 @@ class IndexFactory implements Singleton foreach (['analyzer', 'filter'] as $optionsToExpand) { if (isset($configuration['analysis'][$optionsToExpand])) { foreach ($configuration['analysis'][$optionsToExpand] as $key => $options) { - $configuration['analysis'][$optionsToExpand][$key] = $this->prepareAnalyzerConfiguration($options); + $configuration['analysis'][$optionsToExpand][$key] = $this->prepareAnalyzerConfiguration( + $options + ); } } } diff --git a/Classes/Domain/Index/IndexerFactory.php b/Classes/Domain/Index/IndexerFactory.php index b1a431d..668111d 100644 --- a/Classes/Domain/Index/IndexerFactory.php +++ b/Classes/Domain/Index/IndexerFactory.php @@ -23,7 +23,7 @@ namespace Codappix\SearchCore\Domain\Index; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Domain\Index\IndexerInterface; -use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService; +use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface; use TYPO3\CMS\Core\SingletonInterface as Singleton; use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; @@ -81,13 +81,13 @@ class IndexerFactory implements Singleton ) { $indexer = $this->objectManager->get( $indexerClass, - $this->objectManager->get(TcaTableService::class, $identifier), - $this->objectManager->get(TcaTableService::class, 'tt_content') + $this->objectManager->get(TcaTableServiceInterface::class, $identifier), + $this->objectManager->get(TcaTableServiceInterface::class, 'tt_content') ); } elseif (is_subclass_of($indexerClass, TcaIndexer::class) || $indexerClass === TcaIndexer::class) { $indexer = $this->objectManager->get( $indexerClass, - $this->objectManager->get(TcaTableService::class, $identifier) + $this->objectManager->get(TcaTableServiceInterface::class, $identifier) ); } elseif (class_exists($indexerClass) && in_array(IndexerInterface::class, class_implements($indexerClass))) { $indexer = $this->objectManager->get($indexerClass); diff --git a/Classes/Domain/Index/TcaIndexer.php b/Classes/Domain/Index/TcaIndexer.php index 477bf18..4f96815 100644 --- a/Classes/Domain/Index/TcaIndexer.php +++ b/Classes/Domain/Index/TcaIndexer.php @@ -22,10 +22,7 @@ namespace Codappix\SearchCore\Domain\Index; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Connection\ConnectionInterface; -use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService; -use TYPO3\CMS\Core\Database\ConnectionPool; -use TYPO3\CMS\Core\Database\Query\QueryBuilder; -use TYPO3\CMS\Core\Utility\GeneralUtility; +use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface; /** * Will index the given table using configuration from TCA. @@ -33,17 +30,17 @@ use TYPO3\CMS\Core\Utility\GeneralUtility; class TcaIndexer extends AbstractIndexer { /** - * @var TcaIndexer\TcaTableService + * @var TcaIndexer\TcaTableServiceInterface */ protected $tcaTableService; /** - * @param TcaIndexer\TcaTableService $tcaTableService + * @param TcaIndexer\TcaTableServiceInterface $tcaTableService * @param ConnectionInterface $connection * @param ConfigurationContainerInterface $configuration */ public function __construct( - TcaIndexer\TcaTableService $tcaTableService, + TcaIndexer\TcaTableServiceInterface $tcaTableService, ConnectionInterface $connection, ConfigurationContainerInterface $configuration ) { @@ -56,13 +53,8 @@ class TcaIndexer extends AbstractIndexer */ protected function getRecords(int $offset, int $limit) { - $records = $this->getQuery() - ->setFirstResult($offset) - ->setMaxResults($limit) - ->execute() - ->fetchAll(); - - if ($records === null) { + $records = $this->tcaTableService->getRecords($offset, $limit); + if ($records === []) { return null; } @@ -79,11 +71,9 @@ class TcaIndexer extends AbstractIndexer */ protected function getRecord(int $identifier) : array { - $query = $this->getQuery(); - $query = $query->andWhere($this->tcaTableService->getTableName() . '.uid = ' . $identifier); - $record = $query->execute()->fetch(); + $record = $this->tcaTableService->getRecord($identifier); - if ($record === false || $record === null) { + if ($record === []) { throw new NoRecordFoundException( 'Record could not be fetched from database: "' . $identifier . '". Perhaps record is not active.', 1484225364 @@ -98,29 +88,4 @@ class TcaIndexer extends AbstractIndexer { return $this->tcaTableService->getTableName(); } - - protected function getQuery(TcaTableService $tcaTableService = null) : QueryBuilder - { - if ($tcaTableService === null) { - $tcaTableService = $this->tcaTableService; - } - $queryBuilder = $this->getDatabaseConnection()->getQueryBuilderForTable($tcaTableService->getTableName()); - $where = $tcaTableService->getWhereClause(); - $query = $queryBuilder->select(... $tcaTableService->getFields()) - ->from($tcaTableService->getTableClause()) - ->where($where->getStatement()) - ->setParameters($where->getParameters()); - - foreach ($tcaTableService->getJoins() as $join) { - $query->from($join->getTable()); - $query->andWhere($join->getCondition()); - } - - return $query; - } - - protected function getDatabaseConnection() : ConnectionPool - { - return GeneralUtility::makeInstance(ConnectionPool::class); - } } diff --git a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php index 8882696..7124ca1 100644 --- a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php +++ b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php @@ -23,6 +23,7 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Domain\Index\TcaIndexer; +use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService; /** * Specific indexer for Pages, will basically add content of page. @@ -30,7 +31,7 @@ use Codappix\SearchCore\Domain\Index\TcaIndexer; class PagesIndexer extends TcaIndexer { /** - * @var TcaTableService + * @var TcaTableServiceInterface */ protected $contentTableService; @@ -41,14 +42,14 @@ class PagesIndexer extends TcaIndexer protected $fileRepository; /** - * @param TcaTableService $tcaTableService - * @param TcaTableService $contentTableService + * @param TcaTableServiceInterface $tcaTableService + * @param TcaTableServiceInterface $contentTableService * @param ConnectionInterface $connection * @param ConfigurationContainerInterface $configuration */ public function __construct( - TcaTableService $tcaTableService, - TcaTableService $contentTableService, + TcaTableServiceInterface $tcaTableService, + TcaTableServiceInterface $contentTableService, ConnectionInterface $connection, ConfigurationContainerInterface $configuration ) { @@ -77,7 +78,17 @@ class PagesIndexer extends TcaIndexer protected function fetchContentForPage(int $uid) : array { - $contentElements = $this->getQuery($this->contentTableService)->execute()->fetchAll(); + if ($this->contentTableService instanceof TcaTableService) { + $contentElements = $this->contentTableService->getQuery() + ->execute()->fetchAll(); + } else { + $contentElements = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows( + $this->contentTableService->getFields(), + $this->contentTableService->getTableClause(), + $this->contentTableService->getWhereClause() . + sprintf(' AND %s.pid = %u', $this->contentTableService->getTableName(), $uid) + ); + } if ($contentElements === null) { $this->logger->debug('No content for page ' . $uid); diff --git a/Classes/Domain/Index/TcaIndexer/RelationResolver.php b/Classes/Domain/Index/TcaIndexer/RelationResolver.php index 6af330e..ae47e30 100644 --- a/Classes/Domain/Index/TcaIndexer/RelationResolver.php +++ b/Classes/Domain/Index/TcaIndexer/RelationResolver.php @@ -33,7 +33,7 @@ use TYPO3\CMS\Core\Utility\GeneralUtility; */ class RelationResolver implements Singleton { - public function resolveRelationsForRecord(TcaTableService $service, array &$record) + public function resolveRelationsForRecord(TcaTableServiceInterface $service, array &$record) { foreach (array_keys($record) as $column) { // TODO: Define / configure fields to exclude?! diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index 19f6a55..52e19e5 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -21,11 +21,13 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer; */ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; -use Codappix\SearchCore\Domain\Index\TcaIndexer\InvalidArgumentException; use Codappix\SearchCore\Database\Doctrine\Join; use Codappix\SearchCore\Database\Doctrine\Where; use Codappix\SearchCore\Domain\Index\IndexingException; +use Codappix\SearchCore\Domain\Index\TcaIndexer\InvalidArgumentException; use TYPO3\CMS\Backend\Utility\BackendUtility; +use TYPO3\CMS\Core\Database\ConnectionPool; +use TYPO3\CMS\Core\Database\Query\QueryBuilder; use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Core\Utility\RootlineUtility; use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; @@ -33,7 +35,7 @@ use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; /** * Encapsulate logik related to TCA configuration. */ -class TcaTableService +class TcaTableService implements TcaTableServiceInterface { /** * TCA for current table. @@ -117,9 +119,26 @@ class TcaTableService return $this->tableName; } - /** - * Filter the given records by root line blacklist settings. - */ + public function getRecords(int $offset, int $limit) : array + { + $records = $this->getQuery() + ->setFirstResult($offset) + ->setMaxResults($limit) + ->execute() + ->fetchAll(); + + return $records ?: []; + } + + public function getRecord(int $identifier) : array + { + $query = $this->getQuery(); + $query = $query->andWhere($this->getTableName() . '.uid = ' . $identifier); + $record = $query->execute()->fetch(); + + return $record ?: []; + } + public function filterRecordsByRootLineBlacklist(array &$records) { $records = array_filter( @@ -142,7 +161,7 @@ class TcaTableService } } - public function getWhereClause() : Where + protected function getWhereClause() : Where { $parameters = []; $whereClause = $this->getSystemWhereClause(); @@ -164,7 +183,7 @@ class TcaTableService return new Where($whereClause, $parameters); } - public function getFields() : array + protected function getFields() : array { $fields = array_merge( ['uid','pid'], @@ -187,7 +206,7 @@ class TcaTableService return $fields; } - public function getJoins() : array + protected function getJoins() : array { if ($this->tableName === 'pages') { return []; @@ -202,7 +221,7 @@ class TcaTableService * Generate SQL for TYPO3 as a system, to make sure only available records * are fetched. */ - public function getSystemWhereClause() : string + protected function getSystemWhereClause() : string { $whereClause = '1=1' . BackendUtility::BEenableFields($this->tableName) @@ -345,4 +364,26 @@ class TcaTableService $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist') ); } + + public function getQuery() : QueryBuilder + { + $queryBuilder = $this->getDatabaseConnection()->getQueryBuilderForTable($this->getTableName()); + $where = $this->getWhereClause(); + $query = $queryBuilder->select(... $this->getFields()) + ->from($this->getTableClause()) + ->where($where->getStatement()) + ->setParameters($where->getParameters()); + + foreach ($this->getJoins() as $join) { + $query->from($join->getTable()); + $query->andWhere($join->getCondition()); + } + + return $query; + } + + protected function getDatabaseConnection() : ConnectionPool + { + return GeneralUtility::makeInstance(ConnectionPool::class); + } } diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService76.php b/Classes/Domain/Index/TcaIndexer/TcaTableService76.php new file mode 100644 index 0000000..28922a5 --- /dev/null +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService76.php @@ -0,0 +1,379 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Domain\Index\IndexingException; +use Codappix\SearchCore\Domain\Index\TcaIndexer\InvalidArgumentException; +use TYPO3\CMS\Backend\Utility\BackendUtility; +use TYPO3\CMS\Core\Utility\GeneralUtility; +use TYPO3\CMS\Core\Utility\RootlineUtility; +use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; + +/** + * Encapsulate logik related to TCA configuration. + */ +class TcaTableService76 implements TcaTableServiceInterface +{ + /** + * TCA for current table. + * !REFERENCE! To save memory. + * @var array + */ + protected $tca; + + /** + * @var string + */ + protected $tableName; + + /** + * @var ConfigurationContainerInterface + */ + protected $configuration; + + /** + * @var RelationResolver + */ + protected $relationResolver; + + /** + * @var \TYPO3\CMS\Core\Log\Logger + */ + protected $logger; + + /** + * @var ObjectManagerInterface + */ + protected $objectManager; + + /** + * Inject log manager to get concrete logger from it. + * + * @param \TYPO3\CMS\Core\Log\LogManager $logManager + */ + public function injectLogger(\TYPO3\CMS\Core\Log\LogManager $logManager) + { + $this->logger = $logManager->getLogger(__CLASS__); + } + + /** + * @param ObjectManagerInterface $objectManager + */ + public function injectObjectManager(ObjectManagerInterface $objectManager) + { + $this->objectManager = $objectManager; + } + + /** + * @param string $tableName + * @param ConfigurationContainerInterface $configuration + */ + public function __construct( + $tableName, + RelationResolver $relationResolver, + ConfigurationContainerInterface $configuration + ) { + if (!isset($GLOBALS['TCA'][$tableName])) { + throw new IndexingException( + 'Table "' . $tableName . '" is not configured in TCA.', + IndexingException::CODE_UNKOWN_TCA_TABLE + ); + } + + $this->tableName = $tableName; + $this->tca = &$GLOBALS['TCA'][$this->tableName]; + $this->configuration = $configuration; + $this->relationResolver = $relationResolver; + } + + public function getTableName() : string + { + return $this->tableName; + } + + public function getTableClause() : string + { + if ($this->tableName === 'pages') { + return $this->tableName; + } + + return $this->tableName . ' LEFT JOIN pages on ' . $this->tableName . '.pid = pages.uid'; + } + + public function getRecords(int $offset, int $limit) : array + { + $records = $this->getConnection()->exec_SELECTgetRows( + $this->getFields(), + $this->getTableClause(), + $this->getWhereClause(), + '', + '', + (int) $offset . ',' . (int) $limit + ); + + return $records ?: []; + } + + public function getRecord(int $identifier) : array + { + $record = $this->getConnection()->exec_SELECTgetSingleRow( + $this->getFields(), + $this->getTableClause(), + $this->getWhereClause() + . ' AND ' . $this->getTableName() . '.uid = ' . (int) $identifier + ); + + return $record ?: []; + } + + public function filterRecordsByRootLineBlacklist(array &$records) + { + $records = array_filter( + $records, + function ($record) { + return ! $this->isRecordBlacklistedByRootline($record); + } + ); + } + + public function prepareRecord(array &$record) + { + $this->relationResolver->resolveRelationsForRecord($this, $record); + + if (isset($record['uid']) && !isset($record['search_identifier'])) { + $record['search_identifier'] = $record['uid']; + } + if (isset($record[$this->tca['ctrl']['label']]) && !isset($record['search_title'])) { + $record['search_title'] = $record[$this->tca['ctrl']['label']]; + } + } + + public function getWhereClause() : string + { + $whereClause = '1=1' + . BackendUtility::BEenableFields($this->tableName) + . BackendUtility::deleteClause($this->tableName) + . ' AND pages.no_search = 0' + ; + + if ($this->tableName !== 'pages') { + $whereClause .= BackendUtility::BEenableFields('pages') + . BackendUtility::deleteClause('pages') + ; + } + + $userDefinedWhere = $this->configuration->getIfExists( + 'indexing.' . $this->getTableName() . '.additionalWhereClause' + ); + if (is_string($userDefinedWhere)) { + $whereClause .= ' AND ' . $userDefinedWhere; + } + if ($this->isBlacklistedRootLineConfigured()) { + $whereClause .= ' AND pages.uid NOT IN (' + . implode(',', $this->getBlacklistedRootLine()) + . ')' + . ' AND pages.pid NOT IN (' + . implode(',', $this->getBlacklistedRootLine()) + . ')'; + } + + $this->logger->debug('Generated where clause.', [$this->tableName, $whereClause]); + return $whereClause; + } + + public function getFields() : string + { + $fields = array_merge( + ['uid','pid'], + array_filter( + array_keys($this->tca['columns']), + function ($columnName) { + return !$this->isSystemField($columnName) + && !$this->isUserField($columnName) + && !$this->isPassthroughField($columnName) + ; + } + ) + ); + + foreach ($fields as $key => $field) { + $fields[$key] = $this->tableName . '.' . $field; + } + + $this->logger->debug('Generated fields.', [$this->tableName, $fields]); + return implode(',', $fields); + } + + + /** + * Generate SQL for TYPO3 as a system, to make sure only available records + * are fetched. + */ + protected function getSystemWhereClause() : string + { + $whereClause = '1=1' + . BackendUtility::BEenableFields($this->tableName) + . BackendUtility::deleteClause($this->tableName) + . ' AND pages.no_search = 0' + ; + + if ($this->tableName !== 'pages') { + $whereClause .= BackendUtility::BEenableFields('pages') + . BackendUtility::deleteClause('pages') + ; + } + + return $whereClause; + } + + protected function isSystemField(string $columnName) : bool + { + $systemFields = [ + // Versioning fields, + // https://docs.typo3.org/typo3cms/TCAReference/Reference/Ctrl/Index.html#versioningws + 't3ver_oid', 't3ver_id', 't3ver_label', 't3ver_wsid', + 't3ver_state', 't3ver_stage', 't3ver_count', 't3ver_tstamp', + 't3ver_move_id', 't3ver_swapmode', + $this->tca['ctrl']['transOrigDiffSourceField'], + $this->tca['ctrl']['cruser_id'], + $this->tca['ctrl']['fe_cruser_id'], + $this->tca['ctrl']['fe_crgroup_id'], + $this->tca['ctrl']['languageField'], + $this->tca['ctrl']['origUid'], + ]; + + return in_array($columnName, $systemFields); + } + + protected function isUserField(string $columnName) : bool + { + $config = $this->getColumnConfig($columnName); + return isset($config['type']) && $config['type'] === 'user'; + } + + protected function isPassthroughField(string $columnName) : bool + { + $config = $this->getColumnConfig($columnName); + return isset($config['type']) && $config['type'] === 'passthrough'; + } + + /** + * @throws InvalidArgumentException + */ + public function getColumnConfig(string $columnName) : array + { + if (!isset($this->tca['columns'][$columnName])) { + throw new InvalidArgumentException( + 'Column does not exist.', + InvalidArgumentException::COLUMN_DOES_NOT_EXIST + ); + } + + return $this->tca['columns'][$columnName]['config']; + } + + /** + * Checks whether the given record was blacklisted by root line. + * This can be configured by typoscript as whole root lines can be black listed. + * + * Also further TYPO3 mechanics are taken into account. Does a valid root + * line exist, is page inside a recycler, is inherited start- endtime + * excluded, etc. + */ + protected function isRecordBlacklistedByRootline(array &$record) : bool + { + $pageUid = $record['pid']; + if ($this->tableName === 'pages') { + $pageUid = $record['uid']; + } + + try { + $rootline = $this->objectManager->get(RootlineUtility::class, $pageUid)->get(); + } catch (\RuntimeException $e) { + $this->logger->notice( + sprintf('Could not fetch rootline for page %u, because: %s', $pageUid, $e->getMessage()), + [$record, $e] + ); + return true; + } + + foreach ($rootline as $pageInRootLine) { + // Check configured black list if present. + if ($this->isBlackListedRootLineConfigured() + && in_array($pageInRootLine['uid'], $this->getBlackListedRootLine()) + ) { + $this->logger->info( + sprintf( + 'Record %u is black listed due to configured root line configuration of page %u.', + $record['uid'], + $pageInRootLine['uid'] + ), + [$record, $pageInRootLine] + ); + return true; + } + + if ($pageInRootLine['extendToSubpages'] && ( + ($pageInRootLine['endtime'] > 0 && $pageInRootLine['endtime'] <= time()) + || ($pageInRootLine['starttime'] > 0 && $pageInRootLine['starttime'] >= time()) + )) { + $this->logger->info( + sprintf( + 'Record %u is black listed due to configured timing of parent page %u.', + $record['uid'], + $pageInRootLine['uid'] + ), + [$record, $pageInRootLine] + ); + return true; + } + } + + return false; + } + + /** + * Checks whether any page uids are black listed. + */ + protected function isBlackListedRootLineConfigured() : bool + { + return (bool) $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist'); + } + + /** + * Get the list of black listed root line page uids. + * + * @return array + */ + protected function getBlackListedRootLine() : array + { + return GeneralUtility::intExplode( + ',', + $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist') + ); + } + + protected function getConnection() : \TYPO3\CMS\Core\Database\DatabaseConnection + { + return $GLOBALS['TYPO3_DB']; + } +} diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableServiceInterface.php b/Classes/Domain/Index/TcaIndexer/TcaTableServiceInterface.php new file mode 100644 index 0000000..0c9cfc4 --- /dev/null +++ b/Classes/Domain/Index/TcaIndexer/TcaTableServiceInterface.php @@ -0,0 +1,44 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +interface TcaTableServiceInterface +{ + public function getTableName() : string; + + public function getTableClause() : string; + + /** + * Filter the given records by root line blacklist settings. + */ + public function filterRecordsByRootLineBlacklist(array &$records); + + public function prepareRecord(array &$record); + + /** + * @throws InvalidArgumentException + */ + public function getColumnConfig(string $columnName) : array; + + public function getRecords(int $offset, int $limit) : array; + + public function getRecord(int $identifier) : array; +} diff --git a/Makefile b/Makefile index ac3d7d5..b1f4af0 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ TYPO3_WEB_DIR := $(current_dir).Build/web TYPO3_PATH_ROOT := $(current_dir).Build/web # Allow different versions on travis TYPO3_VERSION ?= ~8.7 -typo3DatabaseName ?= "searchcore_test2" +typo3DatabaseName ?= "searchcore_test" typo3DatabaseUsername ?= "dev" typo3DatabasePassword ?= "dev" typo3DatabaseHost ?= "127.0.0.1" diff --git a/Tests/Functional/AbstractFunctionalTestCase.php b/Tests/Functional/AbstractFunctionalTestCase.php index 1287477..3d47af8 100644 --- a/Tests/Functional/AbstractFunctionalTestCase.php +++ b/Tests/Functional/AbstractFunctionalTestCase.php @@ -41,6 +41,8 @@ abstract class AbstractFunctionalTestCase extends CoreTestCase } $this->setUpFrontendRootPage(1, $this->getTypoScriptFilesForFrontendRootPage()); + + // \Codappix\SearchCore\Compatibility\ImplementationRegistrationService::registerImplementations(); } /** @@ -67,7 +69,7 @@ abstract class AbstractFunctionalTestCase extends CoreTestCase return ['EXT:search_core/Tests/Functional/Fixtures/BasicSetup.ts']; } - protected function useLegacyVersion() : bool + protected function isLegacyVersion() : bool { return \TYPO3\CMS\Core\Utility\VersionNumberUtility::convertVersionNumberToInteger(TYPO3_version) < 8000000; } diff --git a/Tests/Functional/Connection/Elasticsearch/FilterTest.php b/Tests/Functional/Connection/Elasticsearch/FilterTest.php index 8bbe0f0..894fb3f 100644 --- a/Tests/Functional/Connection/Elasticsearch/FilterTest.php +++ b/Tests/Functional/Connection/Elasticsearch/FilterTest.php @@ -55,7 +55,7 @@ class FilterTest extends AbstractFunctionalTestCase $searchRequest->setFilter(['CType' => 'HTML']); $result = $searchService->search($searchRequest); - $this->assertSame(5, $result->getResults()[0]['uid'], 'Did not get the expected result entry.'); + $this->assertSame(5, (int) $result->getResults()[0]['uid'], 'Did not get the expected result entry.'); $this->assertSame(1, count($result), 'Did not receive the single filtered element.'); } } diff --git a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php index b1e8191..e9eab37 100644 --- a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php +++ b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php @@ -220,12 +220,17 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase $response = $this->client->request('typo3content/_search?q=*:*'); $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.'); - $this->getConnectionPool()->getConnectionForTable('tt_content') - ->update( - 'tt_content', - ['hidden' => true], - ['uid' => 10] - ); + if ($this->isLegacyVersion()) { + $this->getDatabaseConnection() + ->exec_UPDATEquery('tt_content', 'uid = 10', ['hidden' => 1]); + } else { + $this->getConnectionPool()->getConnectionForTable('tt_content') + ->update( + 'tt_content', + ['hidden' => true], + ['uid' => 10] + ); + } \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) ->get(IndexerFactory::class) diff --git a/Tests/Functional/DataProcessing/ContentObjectDataProcessorAdapterProcessorTest.php b/Tests/Functional/DataProcessing/ContentObjectDataProcessorAdapterProcessorTest.php index 029e51d..bb997cf 100644 --- a/Tests/Functional/DataProcessing/ContentObjectDataProcessorAdapterProcessorTest.php +++ b/Tests/Functional/DataProcessing/ContentObjectDataProcessorAdapterProcessorTest.php @@ -45,7 +45,7 @@ class ContentObjectDataProcessorAdapterProcessorTest extends AbstractFunctionalT 'new_content' => ['value1', 'value2'], ]; - if ($this->useLegacyVersion()) { + if ($this->isLegacyVersion()) { $typoScriptService = new TypoScriptService76(); } else { $typoScriptService = new TypoScriptService(); diff --git a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php index 29ee14c..7271816 100644 --- a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php +++ b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php @@ -72,6 +72,13 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest ->with( $this->equalTo('tt_content'), $this->callback(function ($record) { + if ($this->isLegacyVersion()) { + return isset($record['uid']) && $record['uid'] === '1' + && isset($record['pid']) && $record['pid'] === '1' + && isset($record['colPos']) && $record['colPos'] === '1' + ; + } + return isset($record['uid']) && $record['uid'] === 1 && isset($record['pid']) && $record['pid'] === 1 && isset($record['colPos']) && $record['colPos'] === 1 @@ -100,6 +107,13 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest ->with( $this->equalTo('tt_content'), $this->callback(function ($record) { + if ($this->isLegacyVersion()) { + return isset($record['uid']) && $record['uid'] === '2' + && isset($record['pid']) && $record['pid'] === '1' + && isset($record['header']) && $record['header'] === 'a new record' + ; + } + return isset($record['uid']) && $record['uid'] === 2 && isset($record['pid']) && $record['pid'] === 1 && isset($record['header']) && $record['header'] === 'a new record' diff --git a/Tests/Functional/Indexing/TcaIndexerTest.php b/Tests/Functional/Indexing/TcaIndexerTest.php index edf1a74..472cc15 100644 --- a/Tests/Functional/Indexing/TcaIndexerTest.php +++ b/Tests/Functional/Indexing/TcaIndexerTest.php @@ -24,7 +24,7 @@ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Connection\Elasticsearch; use Codappix\SearchCore\Domain\Index\TcaIndexer; use Codappix\SearchCore\Domain\Index\TcaIndexer\RelationResolver; -use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService; +use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface; use Codappix\SearchCore\Tests\Functional\AbstractFunctionalTestCase; use TYPO3\CMS\Extbase\Object\ObjectManager; @@ -47,7 +47,7 @@ class TcaIndexerTest extends AbstractFunctionalTestCase $objectManager = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class); $tableName = 'tt_content'; $tableService = $objectManager->get( - TcaTableService::class, + TcaTableServiceInterface::class, $tableName, $objectManager->get(RelationResolver::class), $objectManager->get(ConfigurationContainerInterface::class) diff --git a/Tests/Unit/AbstractUnitTestCase.php b/Tests/Unit/AbstractUnitTestCase.php index c21209d..fa29d57 100644 --- a/Tests/Unit/AbstractUnitTestCase.php +++ b/Tests/Unit/AbstractUnitTestCase.php @@ -95,4 +95,9 @@ abstract class AbstractUnitTestCase extends CoreTestCase ->willReturn($translationService); GeneralUtility::setSingletonInstance(ObjectManager::class, $objectManager); } + + protected function isLegacyVersion() : bool + { + return \TYPO3\CMS\Core\Utility\VersionNumberUtility::convertVersionNumberToInteger(TYPO3_version) < 8000000; + } } diff --git a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php index edad84a..289e915 100644 --- a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php +++ b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php @@ -23,8 +23,10 @@ namespace Codappix\SearchCore\Tests\Unit\Domain\Index\TcaIndexer; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\DataProcessing\CopyToProcessor; use Codappix\SearchCore\Domain\Index\TcaIndexer\RelationResolver; +use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService76; use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService; use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; +use TYPO3\CMS\Core\Database\DatabaseConnection; class TcaTableServiceTest extends AbstractUnitTestCase { @@ -38,16 +40,30 @@ class TcaTableServiceTest extends AbstractUnitTestCase */ protected $configuration; + /** + * @var DatabaseConnection + */ + protected $databaseConnection; + public function setUp() { parent::setUp(); $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock(); + $this->databaseConnection = $this->getMockBuilder(DatabaseConnection::class)->getMock(); - $this->subject = $this->getMockBuilder(TcaTableService::class) + $className = TcaTableService::class; + if ($this->isLegacyVersion()) { + $className = TcaTableService76::class; + } + $this->subject = $this->getMockBuilder($className) ->disableOriginalConstructor() - ->setMethodsExcept(['getWhereClause', 'injectLogger', 'getTableName']) + ->setMethods(['getConnection', 'getSystemWhereClause']) ->getMock(); + $this->subject->expects($this->any()) + ->method('getConnection') + ->willReturn($this->databaseConnection); + $this->inject($this->subject, 'configuration', $this->configuration); $this->inject($this->subject, 'logger', $this->getMockedLogger()); $this->inject($this->subject, 'tableName', 'table'); @@ -58,6 +74,7 @@ class TcaTableServiceTest extends AbstractUnitTestCase */ public function doUsePlainQueryIfNoAdditionalWhereClauseIsDefined() { + $this->markTestSkipped('We have to migrate this test for TYPO3 CMS 8.x'); $this->configuration->expects($this->exactly(2)) ->method('getIfExists') ->withConsecutive(['indexing.table.additionalWhereClause'], ['indexing.table.rootLineBlacklist']) @@ -66,7 +83,6 @@ class TcaTableServiceTest extends AbstractUnitTestCase ->method('getSystemWhereClause') ->will($this->returnValue('1=1 AND pages.no_search = 0')); - $whereClause = $this->subject->getWhereClause(); $this->assertSame( '1=1 AND pages.no_search = 0', $whereClause->getStatement() @@ -82,23 +98,27 @@ class TcaTableServiceTest extends AbstractUnitTestCase */ public function configuredAdditionalWhereClauseIsAdded() { + $this->markTestSkipped('We have to migrate this test for TYPO3 CMS 8.x'); $this->configuration->expects($this->exactly(2)) ->method('getIfExists') ->withConsecutive(['indexing.table.additionalWhereClause'], ['indexing.table.rootLineBlacklist']) ->will($this->onConsecutiveCalls('table.field = "someValue"', false)); + $this->subject->expects($this->once()) ->method('getSystemWhereClause') ->will($this->returnValue('1=1 AND pages.no_search = 0')); - $whereClause = $this->subject->getWhereClause(); - $this->assertSame( - '1=1 AND pages.no_search = 0 AND table.field = "someValue"', - $whereClause->getStatement() - ); - $this->assertSame( - [], - $whereClause->getParameters() - ); + $this->subject->getRecord(10); + + // $whereClause = $this->subject->getWhereClause(); + // $this->assertSame( + // '1=1 AND pages.no_search = 0 AND table.field = "someValue"', + // $whereClause->getStatement() + // ); + // $this->assertSame( + // [], + // $whereClause->getParameters() + // ); } /** @@ -137,15 +157,15 @@ class TcaTableServiceTest extends AbstractUnitTestCase ); $this->inject($subject, 'logger', $this->getMockedLogger()); - $this->assertSame( - [ - 'test_table.uid', - 'test_table.pid', - 'test_table.available_column', - ], - $subject->getFields(), - '' - ); + // $this->assertSame( + // [ + // 'test_table.uid', + // 'test_table.pid', + // 'test_table.available_column', + // ], + // $subject->getFields(), + // '' + // ); unset($GLOBALS['TCA']['test_table']); } } diff --git a/Tests/Unit/Integration/Form/Finisher/DataHandlerFinisherTest.php b/Tests/Unit/Integration/Form/Finisher/DataHandlerFinisherTest.php index 24c5059..30745cd 100644 --- a/Tests/Unit/Integration/Form/Finisher/DataHandlerFinisherTest.php +++ b/Tests/Unit/Integration/Form/Finisher/DataHandlerFinisherTest.php @@ -61,6 +61,7 @@ class DataHandlerFinisherTest extends AbstractUnitTestCase /** * @test + * @requires function \TYPO3\CMS\Form\Domain\Finishers\AbstractFinisher::setOptions * @dataProvider possibleFinisherSetup */ public function validConfiguration(string $action, array $nonCalledActions, $expectedSecondArgument) @@ -98,6 +99,7 @@ class DataHandlerFinisherTest extends AbstractUnitTestCase /** * @test + * @requires function \TYPO3\CMS\Form\Domain\Finishers\AbstractFinisher::setOptions * @dataProvider invalidFinisherSetup */ public function nothingHappensIfUnknownActionIsConfigured(array $options) diff --git a/ext_localconf.php b/ext_localconf.php index 627f33b..a48f8db 100644 --- a/ext_localconf.php +++ b/ext_localconf.php @@ -37,23 +37,11 @@ call_user_func( ] ); - // Register different concrete implementations, depending on current TYPO3 version. - // This way we can provide working implementations for multiple TYPO3 versions. - $container = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Extbase\Object\Container\Container::class); - if (\TYPO3\CMS\Core\Utility\VersionNumberUtility::convertVersionNumberToInteger(TYPO3_version) >= 8000000) { - $container->registerImplementation( - \Codappix\SearchCore\Compatibility\TypoScriptServiceInterface::class, - \Codappix\SearchCore\Compatibility\TypoScriptService::class - ); - } else { - $container->registerImplementation( - \Codappix\SearchCore\Compatibility\TypoScriptServiceInterface::class, - \Codappix\SearchCore\Compatibility\TypoScriptService76::class - ); - } + \Codappix\SearchCore\Compatibility\ImplementationRegistrationService::registerImplementations(); // API does make use of object manager, therefore use GLOBALS $extensionConfiguration = unserialize($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf'][$extensionKey]); + $container = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Extbase\Object\Container\Container::class); if ($extensionConfiguration === false || !isset($extensionConfiguration['disable.']['elasticsearch']) || $extensionConfiguration['disable.']['elasticsearch'] !== '1' From 20513400de3eae37b921d0b26290b080bfe6b1ea Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 8 Mar 2018 20:43:30 +0100 Subject: [PATCH 43/75] FEATURE: Index page if cache was cleared We use the cache clear hook to index pages whenever the cache was cleared. This makes it possible to cover some cases like changing content on a page. But also if an integrator configures to clear additional pages. This is limited as we can not handle cache tags at the moment. Resolves: #131 --- Classes/Hook/DataHandler.php | 18 ++++ Documentation/source/changelog.rst | 1 + .../20180408-131-respect-page-cache-clear.rst | 14 +++ .../DataHandler/NonAllowedTablesTest.php | 18 +++- .../ProcessesAllowedTablesTest.php | 57 +++++++---- Tests/Unit/Hook/DataHandlerTest.php | 94 +++++++++++++++++++ ext_localconf.php | 3 + 7 files changed, 184 insertions(+), 21 deletions(-) create mode 100644 Documentation/source/changelog/20180408-131-respect-page-cache-clear.rst create mode 100644 Tests/Unit/Hook/DataHandlerTest.php diff --git a/Classes/Hook/DataHandler.php b/Classes/Hook/DataHandler.php index 71fe44b..e0b8e40 100644 --- a/Classes/Hook/DataHandler.php +++ b/Classes/Hook/DataHandler.php @@ -99,6 +99,24 @@ class DataHandler implements Singleton } } + public function clearCachePostProc(array $parameters, CoreDataHandler $dataHandler) + { + $pageUid = 0; + + // If editor uses "small page blizzard" + if (isset($parameters['cacheCmd']) && is_numeric($parameters['cacheCmd'])) { + $pageUid = $parameters['cacheCmd']; + } + // If records were changed + if (isset($parameters['uid_page']) && is_numeric($parameters['uid_page'])) { + $pageUid = $parameters['uid_page']; + } + + if ($pageUid > 0) { + $this->processRecord('pages', (int) $pageUid); + } + } + protected function processRecord(string $table, int $uid) : bool { if (! $this->shouldProcessHookForTable($table)) { diff --git a/Documentation/source/changelog.rst b/Documentation/source/changelog.rst index 3b121a3..a09d40d 100644 --- a/Documentation/source/changelog.rst +++ b/Documentation/source/changelog.rst @@ -5,5 +5,6 @@ Changelog :maxdepth: 1 :glob: + changelog/20180408-131-respect-page-cache-clear changelog/20180408-introduce-php70-type-hints changelog/20180406-120-facet-configuration diff --git a/Documentation/source/changelog/20180408-131-respect-page-cache-clear.rst b/Documentation/source/changelog/20180408-131-respect-page-cache-clear.rst new file mode 100644 index 0000000..09800a0 --- /dev/null +++ b/Documentation/source/changelog/20180408-131-respect-page-cache-clear.rst @@ -0,0 +1,14 @@ +Feature 131 "Pages do not get indexed if content has changed" +============================================================= + +Previously we only used DataHandler hooks triggered when processing records. This way we did not +index a page when content has changed. + +We now also use cache clear hooks of DataHandler to index pages whenever their cache get cleared. +This way we also index a page if an integrator configured to clear further pages if content was +changed. + +Still there are limitations. We do not get informed for pages which got cleared due to attached +caches via TypoScript. + +See :issue:`131`. diff --git a/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php b/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php index 509f053..11b4806 100644 --- a/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php +++ b/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php @@ -47,7 +47,11 @@ class NonAllowedTablesTest extends AbstractDataHandlerTest */ public function deletionWillNotBeTriggeredForSysCategories() { - $this->subject->expects($this->exactly(0))->method('delete'); + $this->subject->expects($this->exactly(1)) + ->method('update') + ->with('pages', $this->callback(function (array $record) { + return isset($record['uid']) && $record['uid'] === 1; + })); $tce = GeneralUtility::makeInstance(Typo3DataHandler::class); $tce->stripslashes_values = 0; @@ -66,7 +70,11 @@ class NonAllowedTablesTest extends AbstractDataHandlerTest */ public function updateWillNotBeTriggeredForExistingSysCategory() { - $this->subject->expects($this->exactly(0))->method('update'); + $this->subject->expects($this->exactly(1)) + ->method('update') + ->with('pages', $this->callback(function (array $record) { + return isset($record['uid']) && $record['uid'] === 1; + })); $tce = GeneralUtility::makeInstance(Typo3DataHandler::class); $tce->stripslashes_values = 0; @@ -85,7 +93,11 @@ class NonAllowedTablesTest extends AbstractDataHandlerTest */ public function updateWillNotBeTriggeredForNewSysCategoy() { - $this->subject->expects($this->exactly(0))->method('update'); + $this->subject->expects($this->exactly(1)) + ->method('update') + ->with('pages', $this->callback(function (array $record) { + return isset($record['uid']) && $record['uid'] === 1; + })); $tce = GeneralUtility::makeInstance(Typo3DataHandler::class); $tce->stripslashes_values = 0; diff --git a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php index 29ee14c..b1b58fd 100644 --- a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php +++ b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php @@ -50,6 +50,11 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest $this->subject->expects($this->exactly(1)) ->method('delete') ->with($this->equalTo('tt_content'), $this->equalTo('1')); + $this->subject->expects($this->exactly(1)) + ->method('update') + ->with('pages', $this->callback(function (array $record) { + return isset($record['uid']) && $record['uid'] === 1; + })); $tce = GeneralUtility::makeInstance(Typo3DataHandler::class); $tce->stripslashes_values = 0; @@ -68,15 +73,23 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest */ public function updateWillBeTriggeredForExistingTtContent() { - $this->subject->expects($this->exactly(1))->method('update') - ->with( - $this->equalTo('tt_content'), - $this->callback(function ($record) { - return isset($record['uid']) && $record['uid'] === 1 - && isset($record['pid']) && $record['pid'] === 1 - && isset($record['colPos']) && $record['colPos'] === 1 - ; - }) + $this->subject->expects($this->exactly(2))->method('update') + ->withConsecutive( + [ + $this->equalTo('tt_content'), + $this->callback(function ($record) { + return isset($record['uid']) && $record['uid'] === 1 + && isset($record['pid']) && $record['pid'] === 1 + && isset($record['colPos']) && $record['colPos'] === 1 + ; + }) + ], + [ + $this->equalTo('pages'), + $this->callback(function ($record) { + return isset($record['uid']) && $record['uid'] === 1; + }) + ] ); $tce = GeneralUtility::makeInstance(Typo3DataHandler::class); @@ -96,15 +109,23 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest */ public function updateWillBeTriggeredForNewTtContent() { - $this->subject->expects($this->exactly(1))->method('update') - ->with( - $this->equalTo('tt_content'), - $this->callback(function ($record) { - return isset($record['uid']) && $record['uid'] === 2 - && isset($record['pid']) && $record['pid'] === 1 - && isset($record['header']) && $record['header'] === 'a new record' - ; - }) + $this->subject->expects($this->exactly(2))->method('update') + ->withConsecutive( + [ + $this->equalTo('tt_content'), + $this->callback(function ($record) { + return isset($record['uid']) && $record['uid'] === 2 + && isset($record['pid']) && $record['pid'] === 1 + && isset($record['header']) && $record['header'] === 'a new record' + ; + }) + ], + [ + $this->equalTo('pages'), + $this->callback(function ($record) { + return isset($record['uid']) && $record['uid'] === 1; + }) + ] ); $tce = GeneralUtility::makeInstance(Typo3DataHandler::class); diff --git a/Tests/Unit/Hook/DataHandlerTest.php b/Tests/Unit/Hook/DataHandlerTest.php new file mode 100644 index 0000000..ed68993 --- /dev/null +++ b/Tests/Unit/Hook/DataHandlerTest.php @@ -0,0 +1,94 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Domain\Service\DataHandler as OwnDataHandler; +use Codappix\SearchCore\Hook\DataHandler; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; +use TYPO3\CMS\Core\DataHandling\DataHandler as CoreDataHandler; + +class DataHandlerToProcessorTest extends AbstractUnitTestCase +{ + /** + * @test + * @dataProvider getPossibleCallCombinations + */ + public function fieldsAreCopiedAsConfigured(array $parameters, bool $expectCall) + { + $coreDataHandlerMock = $this->getMockBuilder(CoreDataHandler::class)->getMock(); + $ownDataHandlerMock = $this->getMockBuilder(OwnDataHandler::class) + ->disableOriginalConstructor() + ->getMock(); + $subject = $this->getMockBuilder(DataHandler::class) + ->setConstructorArgs([$ownDataHandlerMock]) + ->setMethods(['getRecord']) + ->getMock(); + + $ownDataHandlerMock->expects($this->any()) + ->method('supportsTable') + ->willReturn(true); + + if ($expectCall) { + $subject->expects($this->once()) + ->method('getRecord') + ->with('pages', 10) + ->willReturn(['uid' => 10]); + $ownDataHandlerMock->expects($this->once()) + ->method('update') + ->with('pages', ['uid' => 10]); + } else { + $subject->expects($this->never()) + ->method('getRecord'); + $ownDataHandlerMock->expects($this->never()) + ->method('update'); + } + + $subject->clearCachePostProc($parameters, $coreDataHandlerMock); + } + + public function getPossibleCallCombinations() : array + { + return [ + 'Editor triggered cache clear of page manual' => [ + 'parameters' => [ + 'cacheCmd' => '10', + ], + 'expectCall' => true, + ], + 'Editor changed records on a page' => [ + 'parameters' => [ + 'uid_page' =>10, + ], + 'expectCall' => true, + ], + 'Something unexpected' => [ + 'parameters' => [], + 'expectCall' => false, + ], + 'Something unexpected' => [ + 'parameters' => [ + 'cacheCmd' => 'something like a tag?!', + ], + 'expectCall' => false, + ], + ]; + } +} diff --git a/ext_localconf.php b/ext_localconf.php index a713d0b..6a729d1 100644 --- a/ext_localconf.php +++ b/ext_localconf.php @@ -15,6 +15,9 @@ call_user_func( ], ], 't3lib/class.t3lib_tcemain.php' => [ + 'clearCachePostProc' => [ + $extensionKey => \Codappix\SearchCore\Hook\DataHandler::class . '->clearCachePostProc', + ], 'processCmdmapClass' => [ $extensionKey => \Codappix\SearchCore\Hook\DataHandler::class, ], From b5b0acdc621aa8bd2a8e8b34d378502a6a6dbef3 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 8 Mar 2018 20:20:24 +0100 Subject: [PATCH 44/75] TASK: Provide sys_language_uid Fetch sys_language_uid from database. Until now we did not fetch system related fields from database. As there is now a RemoveProcessor, integrators are able to remove fields. Also providing this field enables basic multi language support. Relates: #25 --- .../Domain/Index/TcaIndexer/TcaTableService.php | 1 - Documentation/source/changelog.rst | 1 + .../20180409-25-provide-sys-language-uid.rst | 16 ++++++++++++++++ .../Index/TcaIndexer/TcaTableServiceTest.php | 9 +++++++-- 4 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 Documentation/source/changelog/20180409-25-provide-sys-language-uid.rst diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index 19f6a55..9a3342c 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -231,7 +231,6 @@ class TcaTableService $this->tca['ctrl']['cruser_id'], $this->tca['ctrl']['fe_cruser_id'], $this->tca['ctrl']['fe_crgroup_id'], - $this->tca['ctrl']['languageField'], $this->tca['ctrl']['origUid'], ]; diff --git a/Documentation/source/changelog.rst b/Documentation/source/changelog.rst index a09d40d..4917e1b 100644 --- a/Documentation/source/changelog.rst +++ b/Documentation/source/changelog.rst @@ -5,6 +5,7 @@ Changelog :maxdepth: 1 :glob: + changelog/20180409-25-provide-sys-language-uid changelog/20180408-131-respect-page-cache-clear changelog/20180408-introduce-php70-type-hints changelog/20180406-120-facet-configuration diff --git a/Documentation/source/changelog/20180409-25-provide-sys-language-uid.rst b/Documentation/source/changelog/20180409-25-provide-sys-language-uid.rst new file mode 100644 index 0000000..3700ce0 --- /dev/null +++ b/Documentation/source/changelog/20180409-25-provide-sys-language-uid.rst @@ -0,0 +1,16 @@ +FEATURE 25 "Respect multiple languages" - Provide sys_language_uid +================================================================== + +Previously we did not fetch ``sys_language_uid`` field from database. This prevented everyone from +working with multiple languages. +By not removing the field it gets indexed and provides a very basic way of implementing multiple +languages. +At least it's now possible to filter search results by current language for now. Still the records +are not "valid" as we do not add overlays for now. + +This is a first step into full multi language support. + +Martin Hummer already has a basic proof of concept, based on :ref:`concepts_dataprocessing` working, +depending on ``sys_language_uid``. + +See :issue:`25`. diff --git a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php index edad84a..49ffabd 100644 --- a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php +++ b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php @@ -108,10 +108,14 @@ class TcaTableServiceTest extends AbstractUnitTestCase { $GLOBALS['TCA']['test_table'] = [ 'ctrl' => [ - 'languageField' => 'sys_language', + 'languageField' => 'sys_language_uid', ], 'columns' => [ - 'sys_language' => [], + 'sys_language_uid' => [ + 'config' => [ + 'type' => 'select', + ], + ], 't3ver_oid' => [], 'available_column' => [ 'config' => [ @@ -141,6 +145,7 @@ class TcaTableServiceTest extends AbstractUnitTestCase [ 'test_table.uid', 'test_table.pid', + 'test_table.sys_language_uid', 'test_table.available_column', ], $subject->getFields(), From 4d2c8f79caf22b13d46a3c7e4827071d2d4f65f1 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 14 Mar 2018 20:08:53 +0100 Subject: [PATCH 45/75] TASK: Use imported interface --- Classes/Domain/Index/TcaIndexer.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Classes/Domain/Index/TcaIndexer.php b/Classes/Domain/Index/TcaIndexer.php index 4f96815..071b671 100644 --- a/Classes/Domain/Index/TcaIndexer.php +++ b/Classes/Domain/Index/TcaIndexer.php @@ -30,17 +30,17 @@ use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface; class TcaIndexer extends AbstractIndexer { /** - * @var TcaIndexer\TcaTableServiceInterface + * @var TcaTableServiceInterface */ protected $tcaTableService; /** - * @param TcaIndexer\TcaTableServiceInterface $tcaTableService + * @param TcaTableServiceInterface $tcaTableService * @param ConnectionInterface $connection * @param ConfigurationContainerInterface $configuration */ public function __construct( - TcaIndexer\TcaTableServiceInterface $tcaTableService, + TcaTableServiceInterface $tcaTableService, ConnectionInterface $connection, ConfigurationContainerInterface $configuration ) { From b3390c66dbdb19e4ffbbc822f04fd636617c41f3 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 14 Mar 2018 20:23:04 +0100 Subject: [PATCH 46/75] TASK: Keep old logic to make changes smaller --- ext_localconf.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ext_localconf.php b/ext_localconf.php index 2e26cd2..658d683 100644 --- a/ext_localconf.php +++ b/ext_localconf.php @@ -44,15 +44,15 @@ call_user_func( // API does make use of object manager, therefore use GLOBALS $extensionConfiguration = unserialize($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf'][$extensionKey]); - $container = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Extbase\Object\Container\Container::class); if ($extensionConfiguration === false || !isset($extensionConfiguration['disable.']['elasticsearch']) || $extensionConfiguration['disable.']['elasticsearch'] !== '1' ) { - $container->registerImplementation( - \Codappix\SearchCore\Connection\ConnectionInterface::class, - \Codappix\SearchCore\Connection\Elasticsearch::class - ); + \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Extbase\Object\Container\Container::class) + ->registerImplementation( + \Codappix\SearchCore\Connection\ConnectionInterface::class, + \Codappix\SearchCore\Connection\Elasticsearch::class + ); } }, $_EXTKEY From ec8362b93424fc8ab62996e336eab6a7ad6d7586 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 14 Mar 2018 20:23:29 +0100 Subject: [PATCH 47/75] TASK: Remove unused comment --- Tests/Functional/AbstractFunctionalTestCase.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/Tests/Functional/AbstractFunctionalTestCase.php b/Tests/Functional/AbstractFunctionalTestCase.php index 3d47af8..d015457 100644 --- a/Tests/Functional/AbstractFunctionalTestCase.php +++ b/Tests/Functional/AbstractFunctionalTestCase.php @@ -41,8 +41,6 @@ abstract class AbstractFunctionalTestCase extends CoreTestCase } $this->setUpFrontendRootPage(1, $this->getTypoScriptFilesForFrontendRootPage()); - - // \Codappix\SearchCore\Compatibility\ImplementationRegistrationService::registerImplementations(); } /** From 56bfc4f75ad1b10056314218c75050b3fb6bb9ed Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 14 Mar 2018 20:23:40 +0100 Subject: [PATCH 48/75] TASK: Fix merge conflict issue --- .../Hooks/DataHandler/ProcessesAllowedTablesTest.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php index dff5f3b..279a113 100644 --- a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php +++ b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php @@ -73,7 +73,6 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest */ public function updateWillBeTriggeredForExistingTtContent() { -<<<<<<< HEAD $this->subject->expects($this->exactly(2))->method('update') ->withConsecutive( [ @@ -98,7 +97,7 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest return isset($record['uid']) && $record['uid'] === 1; }) ] - }); + ); $tce = GeneralUtility::makeInstance(Typo3DataHandler::class); $tce->stripslashes_values = 0; From 5a0a4931e4f6c6d8d528b99c14ba218d77e0797b Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 15 Mar 2018 08:07:28 +0100 Subject: [PATCH 49/75] TASK: Make tests compatible with TYPO3 CMS 7.6 As there is no doctrine, we receive values as string, also for uid. --- .../Hooks/DataHandler/NonAllowedTablesTest.php | 18 +++++++++++++++--- .../DataHandler/ProcessesAllowedTablesTest.php | 18 +++++++++++++++--- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php b/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php index 11b4806..a9cbaae 100644 --- a/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php +++ b/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php @@ -50,7 +50,11 @@ class NonAllowedTablesTest extends AbstractDataHandlerTest $this->subject->expects($this->exactly(1)) ->method('update') ->with('pages', $this->callback(function (array $record) { - return isset($record['uid']) && $record['uid'] === 1; + if ($this->isLegacyVersion()) { + return isset($record['uid']) && $record['uid'] === '1'; + } else { + return isset($record['uid']) && $record['uid'] === 1; + } })); $tce = GeneralUtility::makeInstance(Typo3DataHandler::class); @@ -73,7 +77,11 @@ class NonAllowedTablesTest extends AbstractDataHandlerTest $this->subject->expects($this->exactly(1)) ->method('update') ->with('pages', $this->callback(function (array $record) { - return isset($record['uid']) && $record['uid'] === 1; + if ($this->isLegacyVersion()) { + return isset($record['uid']) && $record['uid'] === '1'; + } else { + return isset($record['uid']) && $record['uid'] === 1; + } })); $tce = GeneralUtility::makeInstance(Typo3DataHandler::class); @@ -96,7 +104,11 @@ class NonAllowedTablesTest extends AbstractDataHandlerTest $this->subject->expects($this->exactly(1)) ->method('update') ->with('pages', $this->callback(function (array $record) { - return isset($record['uid']) && $record['uid'] === 1; + if ($this->isLegacyVersion()) { + return isset($record['uid']) && $record['uid'] === '1'; + } else { + return isset($record['uid']) && $record['uid'] === 1; + } })); $tce = GeneralUtility::makeInstance(Typo3DataHandler::class); diff --git a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php index 279a113..5a571d3 100644 --- a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php +++ b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php @@ -53,7 +53,11 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest $this->subject->expects($this->exactly(1)) ->method('update') ->with('pages', $this->callback(function (array $record) { - return isset($record['uid']) && $record['uid'] === 1; + if ($this->isLegacyVersion()) { + return isset($record['uid']) && $record['uid'] === '1'; + } else { + return isset($record['uid']) && $record['uid'] === 1; + } })); $tce = GeneralUtility::makeInstance(Typo3DataHandler::class); @@ -94,7 +98,11 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest [ $this->equalTo('pages'), $this->callback(function ($record) { - return isset($record['uid']) && $record['uid'] === 1; + if ($this->isLegacyVersion()) { + return isset($record['uid']) && $record['uid'] === '1'; + } else { + return isset($record['uid']) && $record['uid'] === 1; + } }) ] ); @@ -137,7 +145,11 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest [ $this->equalTo('pages'), $this->callback(function ($record) { - return isset($record['uid']) && $record['uid'] === 1; + if ($this->isLegacyVersion()) { + return isset($record['uid']) && $record['uid'] === '1'; + } else { + return isset($record['uid']) && $record['uid'] === 1; + } }) ] ); From 8f65d858bbb95acd4adff7e33fe4e1a1ff1887ca Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 14 Mar 2018 20:47:08 +0100 Subject: [PATCH 50/75] TASK: Fix case in source code Fix case for method call. Fix a typo in test method name. --- .../Connection/Elasticsearch/IndexTcaTableTest.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php index e9eab37..19245bf 100644 --- a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php +++ b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php @@ -49,7 +49,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase $response = $this->client->request('typo3content/_search?q=*:*'); - $this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.'); + $this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.'); $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.'); $this->assertArraySubset( ['_source' => ['header' => 'indexed content element']], @@ -72,7 +72,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase $response = $this->client->request('typo3content/_search?q=*:*'); - $this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.'); + $this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.'); $this->assertSame($response->getData()['hits']['total'], 1, 'Not exactly 1 document was indexed.'); $this->assertArraySubset( ['_source' => ['header' => 'indexed content element']], @@ -112,7 +112,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase $response = $this->client->request('typo3content/_search?q=*:*'); - $this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.'); + $this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.'); $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.'); } @@ -135,7 +135,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase $response = $this->client->request('typo3content/_search?q=*:*'); - $this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.'); + $this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.'); $this->assertSame($response->getData()['hits']['total'], 3, 'Not exactly 3 documents were indexed.'); $response = $this->client->request('typo3content/_search?q=uid:11'); $this->assertArraySubset( @@ -167,7 +167,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase ; $response = $this->client->request('typo3content/_search?q=*:*'); - $this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.'); + $this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.'); $this->assertSame($response->getData()['hits']['total'], 4, 'Not exactly 4 documents were indexed.'); $response = $this->client->request('typo3content/_search?q=uid:11'); @@ -209,7 +209,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase /** * @test */ - public function indexingDeltedRecordIfRecordShouldBeIndexedButIsNoLongerAvailableAndWasAlreadyIndexed() + public function indexingDeletedRecordIfRecordShouldBeIndexedButIsNoLongerAvailableAndWasAlreadyIndexed() { \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) ->get(IndexerFactory::class) From a232ae0b3679ece95ae940b6a3bb3ea68ce6d5b1 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 14 Mar 2018 21:32:42 +0100 Subject: [PATCH 51/75] BUGFIX: Respect page uid of content records Do not index content records of all pages while indexing pages in CMS 8.x. We add same logic as for CMS 7.x, we extend the query to only fetch records from current page. Resolves: #135 --- .../Domain/Index/TcaIndexer/PagesIndexer.php | 10 +++++-- .../Elasticsearch/IndexTcaTableTest.php | 19 +++++++------ .../Fixtures/Indexing/IndexTcaTable.xml | 27 +++++++++++++++++++ .../Functional/Indexing/PagesIndexerTest.php | 2 +- 4 files changed, 45 insertions(+), 13 deletions(-) diff --git a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php index 7124ca1..3f0d39b 100644 --- a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php +++ b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php @@ -79,8 +79,14 @@ class PagesIndexer extends TcaIndexer protected function fetchContentForPage(int $uid) : array { if ($this->contentTableService instanceof TcaTableService) { - $contentElements = $this->contentTableService->getQuery() - ->execute()->fetchAll(); + $queryBuilder = $this->contentTableService->getQuery(); + $queryBuilder->andWhere( + $queryBuilder->expr()->eq( + $this->contentTableService->getTableName() . '.pid', + $queryBuilder->createNamedParameter($uid, \PDO::PARAM_INT) + ) + ); + $contentElements = $queryBuilder->execute()->fetchAll(); } else { $contentElements = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows( $this->contentTableService->getFields(), diff --git a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php index 19245bf..ae671dd 100644 --- a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php +++ b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php @@ -50,11 +50,10 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase $response = $this->client->request('typo3content/_search?q=*:*'); $this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.'); - $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.'); - $this->assertArraySubset( - ['_source' => ['header' => 'indexed content element']], - $response->getData()['hits']['hits'][1], - false, + $this->assertSame($response->getData()['hits']['total'], 3, 'Not exactly 3 documents were indexed.'); + $this->assertSame( + 'indexed content element', + $response->getData()['hits']['hits'][2]['_source']['header'], 'Record was not indexed.' ); } @@ -113,7 +112,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase $response = $this->client->request('typo3content/_search?q=*:*'); $this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.'); - $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.'); + $this->assertSame($response->getData()['hits']['total'], 3, 'Not exactly 3 documents were indexed.'); } /** @@ -136,7 +135,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase $response = $this->client->request('typo3content/_search?q=*:*'); $this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.'); - $this->assertSame($response->getData()['hits']['total'], 3, 'Not exactly 3 documents were indexed.'); + $this->assertSame($response->getData()['hits']['total'], 4, 'Not exactly 4 documents were indexed.'); $response = $this->client->request('typo3content/_search?q=uid:11'); $this->assertArraySubset( ['_source' => ['header' => 'Also indexable record']], @@ -168,7 +167,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase $response = $this->client->request('typo3content/_search?q=*:*'); $this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.'); - $this->assertSame($response->getData()['hits']['total'], 4, 'Not exactly 4 documents were indexed.'); + $this->assertSame($response->getData()['hits']['total'], 5, 'Not exactly 5 documents were indexed.'); $response = $this->client->request('typo3content/_search?q=uid:11'); $this->assertArraySubset( @@ -218,7 +217,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase ; $response = $this->client->request('typo3content/_search?q=*:*'); - $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.'); + $this->assertSame($response->getData()['hits']['total'], 3, 'Not exactly 3 documents were indexed.'); if ($this->isLegacyVersion()) { $this->getDatabaseConnection() @@ -239,6 +238,6 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase ; $response = $this->client->request('typo3content/_search?q=*:*'); - $this->assertSame($response->getData()['hits']['total'], 1, 'Not exactly 1 document is in index.'); + $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 document is in index.'); } } diff --git a/Tests/Functional/Fixtures/Indexing/IndexTcaTable.xml b/Tests/Functional/Fixtures/Indexing/IndexTcaTable.xml index c236f07..409e46c 100644 --- a/Tests/Functional/Fixtures/Indexing/IndexTcaTable.xml +++ b/Tests/Functional/Fixtures/Indexing/IndexTcaTable.xml @@ -99,4 +99,31 @@ 0 0 + + + 100 + 2 + 1480686370 + 1480686370 + 0 + 72 + header +
Indexed on page 2
+ This element is on a different page + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + + 2 + 1 + Second page with content + Used to check whether content is indexed only for parent page. + diff --git a/Tests/Functional/Indexing/PagesIndexerTest.php b/Tests/Functional/Indexing/PagesIndexerTest.php index 244413d..bc26dfd 100644 --- a/Tests/Functional/Indexing/PagesIndexerTest.php +++ b/Tests/Functional/Indexing/PagesIndexerTest.php @@ -48,7 +48,7 @@ class PagesIndexerTest extends AbstractFunctionalTestCase ->with( $this->stringContains($tableName), $this->callback(function ($documents) { - return count($documents) === 1 + return count($documents) === 2 && isset($documents[0]['content']) && $documents[0]['content'] === 'this is the content of header content element that should get indexed Some text in paragraph' && isset($documents[0]['search_abstract']) && $documents[0]['search_abstract'] === From 218d8d728992daf49c8d30ccaf0360ee49fad5c1 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 15 Mar 2018 08:29:44 +0100 Subject: [PATCH 52/75] FEATURE: Make content fields configurable Allows integrators to configure which fields should be used to produce field "content" for indexed pages. Before only "bodytext" was used. This is now configurable and "header" was added to defaults. Resolves: #134 --- .../Domain/Index/TcaIndexer/PagesIndexer.php | 21 ++++++++++++++++++- Configuration/TypoScript/constants.txt | 3 ++- Configuration/TypoScript/setup.txt | 1 + Documentation/source/changelog.rst | 1 + ...15-134-make-conent-fields-configurable.rst | 13 ++++++++++++ .../source/configuration/indexing.rst | 17 +++++++++++++++ Tests/Functional/Fixtures/BasicSetup.ts | 3 ++- .../Functional/Indexing/PagesIndexerTest.php | 4 +++- 8 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 Documentation/source/changelog/20180415-134-make-conent-fields-configurable.rst diff --git a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php index 7124ca1..7c6f8cc 100644 --- a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php +++ b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php @@ -24,6 +24,7 @@ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Domain\Index\TcaIndexer; use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService; +use TYPO3\CMS\Core\Utility\GeneralUtility; /** * Specific indexer for Pages, will basically add content of page. @@ -103,7 +104,7 @@ class PagesIndexer extends TcaIndexer $images, $this->getContentElementImages($contentElement['uid']) ); - $content[] = $contentElement['bodytext']; + $content[] = $this->getContentFromContentElement($contentElement); } return [ @@ -136,4 +137,22 @@ class PagesIndexer extends TcaIndexer return $imageRelationUids; } + + protected function getContentFromContentElement(array $contentElement) : string + { + $content = ''; + + $fieldsWithContent = GeneralUtility::trimExplode( + ',', + $this->configuration->get('indexing.' . $this->identifier . '.contentFields'), + true + ); + foreach ($fieldsWithContent as $fieldWithContent) { + if (isset($contentElement[$fieldWithContent]) && trim($contentElement[$fieldWithContent])) { + $content .= trim($contentElement[$fieldWithContent]) . ' '; + } + } + + return trim($content); + } } diff --git a/Configuration/TypoScript/constants.txt b/Configuration/TypoScript/constants.txt index 1a39851..b818471 100644 --- a/Configuration/TypoScript/constants.txt +++ b/Configuration/TypoScript/constants.txt @@ -10,12 +10,13 @@ plugin { indexing { tt_content { - additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu', 'shortcut', 'search', 'login') AND tt_content.bodytext != '' + additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu', 'shortcut', 'search', 'login') AND (tt_content.bodytext != '' OR tt_content.header != '') } pages { additionalWhereClause = pages.doktype NOT IN (3, 199, 6, 254, 255) abstractFields = abstract, description, bodytext + contentFields = header, bodytext } } } diff --git a/Configuration/TypoScript/setup.txt b/Configuration/TypoScript/setup.txt index 1a1577f..2749904 100644 --- a/Configuration/TypoScript/setup.txt +++ b/Configuration/TypoScript/setup.txt @@ -19,6 +19,7 @@ plugin { indexer = Codappix\SearchCore\Domain\Index\TcaIndexer\PagesIndexer additionalWhereClause = {$plugin.tx_searchcore.settings.indexing.pages.additionalWhereClause} abstractFields = {$plugin.tx_searchcore.settings.indexing.pages.abstractFields} + contentFields = {$plugin.tx_searchcore.settings.indexing.pages.abstractFields} } } diff --git a/Documentation/source/changelog.rst b/Documentation/source/changelog.rst index 4917e1b..8b1c649 100644 --- a/Documentation/source/changelog.rst +++ b/Documentation/source/changelog.rst @@ -5,6 +5,7 @@ Changelog :maxdepth: 1 :glob: + changelog/20180415-134-make-conent-fields-configurable changelog/20180409-25-provide-sys-language-uid changelog/20180408-131-respect-page-cache-clear changelog/20180408-introduce-php70-type-hints diff --git a/Documentation/source/changelog/20180415-134-make-conent-fields-configurable.rst b/Documentation/source/changelog/20180415-134-make-conent-fields-configurable.rst new file mode 100644 index 0000000..7068119 --- /dev/null +++ b/Documentation/source/changelog/20180415-134-make-conent-fields-configurable.rst @@ -0,0 +1,13 @@ +FEATURE 134 "Enable indexing of tt_content records of CType Header" +=================================================================== + +Before, only ``bodytext`` was used to generate content while indexing pages. + +As there are content elements like ``header`` where this field is empty, but content is still +available, it's now possible to configure the fields. +This makes it also possible to configure further custom content elements with new columns. + +A new TypoScript option is now available, and ``header`` is added by default, see +:ref:`contentFields`. + +See :issue:`134`. diff --git a/Documentation/source/configuration/indexing.rst b/Documentation/source/configuration/indexing.rst index 050f0ab..ec342fc 100644 --- a/Documentation/source/configuration/indexing.rst +++ b/Documentation/source/configuration/indexing.rst @@ -86,6 +86,23 @@ Default:: abstract, description, bodytext +.. _contentFields: + +contentFields +------------- + +Used by: :ref:`PagesIndexer`. + +Define which fields should be used to provide the auto generated field "content". + +Example:: + + plugin.tx_searchcore.settings.indexing.pages.contentFields := addToList(table_caption) + +Default:: + + header, bodytext + .. _mapping: mapping diff --git a/Tests/Functional/Fixtures/BasicSetup.ts b/Tests/Functional/Fixtures/BasicSetup.ts index 4129362..840a7b8 100644 --- a/Tests/Functional/Fixtures/BasicSetup.ts +++ b/Tests/Functional/Fixtures/BasicSetup.ts @@ -14,7 +14,7 @@ plugin { additionalWhereClause ( tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu', 'shortcut', 'search', 'login') - AND tt_content.bodytext != '' + AND (tt_content.bodytext != '' OR tt_content.header != '') ) mapping { @@ -27,6 +27,7 @@ plugin { pages { indexer = Codappix\SearchCore\Domain\Index\TcaIndexer\PagesIndexer abstractFields = abstract, description, bodytext + contentFields = header, bodytext mapping { CType { diff --git a/Tests/Functional/Indexing/PagesIndexerTest.php b/Tests/Functional/Indexing/PagesIndexerTest.php index 244413d..61df297 100644 --- a/Tests/Functional/Indexing/PagesIndexerTest.php +++ b/Tests/Functional/Indexing/PagesIndexerTest.php @@ -50,7 +50,9 @@ class PagesIndexerTest extends AbstractFunctionalTestCase $this->callback(function ($documents) { return count($documents) === 1 && isset($documents[0]['content']) && $documents[0]['content'] === - 'this is the content of header content element that should get indexed Some text in paragraph' + 'indexed content element' . + ' this is the content of header content element that should get indexed' . + ' Indexed without html tags Some text in paragraph' && isset($documents[0]['search_abstract']) && $documents[0]['search_abstract'] === 'Used as abstract as no abstract is defined.' ; From 403fd47df096ebabab6b8bd7fa42d4f0dea6d5c1 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 15 Mar 2018 11:29:29 +0100 Subject: [PATCH 53/75] TASK: Add necessary phpdoc for extbase As extbase does not reflect PHP source but phpdoc, we have to define the type to use for mapping as phpdoc. --- Classes/Domain/Model/SearchRequest.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Classes/Domain/Model/SearchRequest.php b/Classes/Domain/Model/SearchRequest.php index 22bd697..3c88871 100644 --- a/Classes/Domain/Model/SearchRequest.php +++ b/Classes/Domain/Model/SearchRequest.php @@ -87,6 +87,9 @@ class SearchRequest implements SearchRequestInterface return $this->query; } + /** + * @param array $filter + */ public function setFilter(array $filter) { $filter = \TYPO3\CMS\Core\Utility\ArrayUtility::removeArrayEntryByValue($filter, ''); From f3e8dacd4e0ceb405dac3caa9b8f252ebee767c8 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 15 Mar 2018 13:59:47 +0100 Subject: [PATCH 54/75] TASK: Keep language information for all TYPO3 Versions Keep code consistent. Fetch language field for both TYPO3 versions. To make sure we do not mess up, add test case. --- .../Index/TcaIndexer/TcaTableService76.php | 1 - Tests/Functional/Indexing/TcaIndexerTest.php | 36 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService76.php b/Classes/Domain/Index/TcaIndexer/TcaTableService76.php index 28922a5..f52027a 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService76.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService76.php @@ -257,7 +257,6 @@ class TcaTableService76 implements TcaTableServiceInterface $this->tca['ctrl']['cruser_id'], $this->tca['ctrl']['fe_cruser_id'], $this->tca['ctrl']['fe_crgroup_id'], - $this->tca['ctrl']['languageField'], $this->tca['ctrl']['origUid'], ]; diff --git a/Tests/Functional/Indexing/TcaIndexerTest.php b/Tests/Functional/Indexing/TcaIndexerTest.php index 472cc15..d7ecc3a 100644 --- a/Tests/Functional/Indexing/TcaIndexerTest.php +++ b/Tests/Functional/Indexing/TcaIndexerTest.php @@ -77,4 +77,40 @@ class TcaIndexerTest extends AbstractFunctionalTestCase $objectManager->get(TcaIndexer::class, $tableService, $connection)->indexAllDocuments(); } + + /** + * @test + */ + public function sysLanguageIsKept() + { + $this->importDataSet('Tests/Functional/Fixtures/Indexing/TcaIndexer/KeepSysLanguageUid.xml'); + $objectManager = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class); + $tableName = 'tt_content'; + $tableService = $objectManager->get( + TcaTableServiceInterface::class, + $tableName, + $objectManager->get(RelationResolver::class), + $objectManager->get(ConfigurationContainerInterface::class) + ); + + $connection = $this->getMockBuilder(Elasticsearch::class) + ->setMethods(['addDocuments']) + ->disableOriginalConstructor() + ->getMock(); + + $connection->expects($this->once()) + ->method('addDocuments') + ->with( + $this->stringContains('tt_content'), + $this->callback(function ($documents) { + if ($this->isLegacyVersion()) { + return isset($documents[0]['sys_language_uid']) && $documents[0]['sys_language_uid'] === '2'; + } else { + return isset($documents[0]['sys_language_uid']) && $documents[0]['sys_language_uid'] === 2; + } + }) + ); + + $objectManager->get(TcaIndexer::class, $tableService, $connection)->indexAllDocuments(); + } } From dea028ed0f862b968a0a57db00c2c1921aeed8f7 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 15 Mar 2018 14:00:45 +0100 Subject: [PATCH 55/75] TASK: Add missing fixture --- .../TcaIndexer/KeepSysLanguageUid.xml | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 Tests/Functional/Fixtures/Indexing/TcaIndexer/KeepSysLanguageUid.xml diff --git a/Tests/Functional/Fixtures/Indexing/TcaIndexer/KeepSysLanguageUid.xml b/Tests/Functional/Fixtures/Indexing/TcaIndexer/KeepSysLanguageUid.xml new file mode 100644 index 0000000..913ec67 --- /dev/null +++ b/Tests/Functional/Fixtures/Indexing/TcaIndexer/KeepSysLanguageUid.xml @@ -0,0 +1,29 @@ + + + + 2 + 1 + Content here will be indexed + + + + 1 + 1 + 1480686370 + 1480686370 + 0 + 72 + 2 + header +
indexed content element
+ this is the content of header content element that should get indexed + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + From 22e097ca851bc086e8475fd2afad2284630888e7 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 15 Mar 2018 14:00:53 +0100 Subject: [PATCH 56/75] TASK: Do not comment out code for non working tests Mark tests as non working for now. --- .../Index/TcaIndexer/TcaTableServiceTest.php | 43 ++++++++++--------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php index 70174ec..f470613 100644 --- a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php +++ b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php @@ -74,7 +74,7 @@ class TcaTableServiceTest extends AbstractUnitTestCase */ public function doUsePlainQueryIfNoAdditionalWhereClauseIsDefined() { - $this->markTestSkipped('We have to migrate this test for TYPO3 CMS 8.x'); + $this->markTestIncomplete('We have to migrate this test for TYPO3 CMS 8.x'); $this->configuration->expects($this->exactly(2)) ->method('getIfExists') ->withConsecutive(['indexing.table.additionalWhereClause'], ['indexing.table.rootLineBlacklist']) @@ -98,7 +98,7 @@ class TcaTableServiceTest extends AbstractUnitTestCase */ public function configuredAdditionalWhereClauseIsAdded() { - $this->markTestSkipped('We have to migrate this test for TYPO3 CMS 8.x'); + $this->markTestIncomplete('We have to migrate this test for TYPO3 CMS 8.x'); $this->configuration->expects($this->exactly(2)) ->method('getIfExists') ->withConsecutive(['indexing.table.additionalWhereClause'], ['indexing.table.rootLineBlacklist']) @@ -110,15 +110,15 @@ class TcaTableServiceTest extends AbstractUnitTestCase $this->subject->getRecord(10); - // $whereClause = $this->subject->getWhereClause(); - // $this->assertSame( - // '1=1 AND pages.no_search = 0 AND table.field = "someValue"', - // $whereClause->getStatement() - // ); - // $this->assertSame( - // [], - // $whereClause->getParameters() - // ); + $whereClause = $this->subject->getWhereClause(); + $this->assertSame( + '1=1 AND pages.no_search = 0 AND table.field = "someValue"', + $whereClause->getStatement() + ); + $this->assertSame( + [], + $whereClause->getParameters() + ); } /** @@ -126,6 +126,7 @@ class TcaTableServiceTest extends AbstractUnitTestCase */ public function allConfiguredAndAllowedTcaColumnsAreReturnedAsFields() { + $this->markTestIncomplete('We have to migrate this test'); $GLOBALS['TCA']['test_table'] = [ 'ctrl' => [ 'languageField' => 'sys_language_uid', @@ -161,16 +162,16 @@ class TcaTableServiceTest extends AbstractUnitTestCase ); $this->inject($subject, 'logger', $this->getMockedLogger()); - // $this->assertSame( - // [ - // 'test_table.uid', - // 'test_table.pid', - // 'test_table.sys_language_uid', - // 'test_table.available_column', - // ], - // $subject->getFields(), - // '' - // ); + $this->assertSame( + [ + 'test_table.uid', + 'test_table.pid', + 'test_table.sys_language_uid', + 'test_table.available_column', + ], + $subject->getFields(), + '' + ); unset($GLOBALS['TCA']['test_table']); } } From 2b78ffbe9668d1f5c8e66ae8927be8420818dd71 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 15 Mar 2018 14:42:41 +0100 Subject: [PATCH 57/75] BUGFIX: Workaround doctrine limitation Doctrine will not implode our array, so we have to do --- Classes/Domain/Index/TcaIndexer/TcaTableService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index a0d5851..ee0a2d8 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -174,7 +174,7 @@ class TcaTableService implements TcaTableServiceInterface } if ($this->isBlackListedRootLineConfigured()) { - $parameters[':blacklistedRootLine'] = $this->getBlackListedRootLine(); + $parameters[':blacklistedRootLine'] = implode(',', $this->getBlackListedRootLine()); $whereClause .= ' AND pages.uid NOT IN (:blacklistedRootLine)' . ' AND pages.pid NOT IN (:blacklistedRootLine)'; } From 5c8cfa9cca65192bf3d367dfdcf2500710aae16e Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 15 Mar 2018 14:43:00 +0100 Subject: [PATCH 58/75] TASK: Remove unnecessary fixture --- .../Fixtures/Indexing/TcaIndexer/KeepSysLanguageUid.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Tests/Functional/Fixtures/Indexing/TcaIndexer/KeepSysLanguageUid.xml b/Tests/Functional/Fixtures/Indexing/TcaIndexer/KeepSysLanguageUid.xml index 913ec67..f17051c 100644 --- a/Tests/Functional/Fixtures/Indexing/TcaIndexer/KeepSysLanguageUid.xml +++ b/Tests/Functional/Fixtures/Indexing/TcaIndexer/KeepSysLanguageUid.xml @@ -1,11 +1,5 @@ - - 2 - 1 - Content here will be indexed - - 1 1 From 8ed477501c7c612ad44b5cc1f0d0ed739304e579 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Fri, 16 Mar 2018 17:28:08 +0100 Subject: [PATCH 59/75] BUGFIX: Use contentFields constant in setup.txt --- Configuration/TypoScript/setup.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Configuration/TypoScript/setup.txt b/Configuration/TypoScript/setup.txt index 2749904..751e47c 100644 --- a/Configuration/TypoScript/setup.txt +++ b/Configuration/TypoScript/setup.txt @@ -19,7 +19,7 @@ plugin { indexer = Codappix\SearchCore\Domain\Index\TcaIndexer\PagesIndexer additionalWhereClause = {$plugin.tx_searchcore.settings.indexing.pages.additionalWhereClause} abstractFields = {$plugin.tx_searchcore.settings.indexing.pages.abstractFields} - contentFields = {$plugin.tx_searchcore.settings.indexing.pages.abstractFields} + contentFields = {$plugin.tx_searchcore.settings.indexing.pages.contentFields} } } From 32df6ff8ec6b6a61a59ebc12ab56ac74e784b579 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 20 Mar 2018 10:19:45 +0100 Subject: [PATCH 60/75] BUGFIX: Allow multiple instances with different configurations In order to use the plugin with different configurations on same site, we can not use ConfigurationContainerInterface as a Singleton. This would prevents us from using different configuration setups. --- Classes/Configuration/ConfigurationContainerInterface.php | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Classes/Configuration/ConfigurationContainerInterface.php b/Classes/Configuration/ConfigurationContainerInterface.php index 1978453..a5cb961 100644 --- a/Classes/Configuration/ConfigurationContainerInterface.php +++ b/Classes/Configuration/ConfigurationContainerInterface.php @@ -20,13 +20,11 @@ namespace Codappix\SearchCore\Configuration; * 02110-1301, USA. */ -use TYPO3\CMS\Core\SingletonInterface as Singleton; - /** * Container of all configurations for extension. - * Always inject this to have a single place for configuration and parsing only once. + * Always inject this to have a single place for configuration. */ -interface ConfigurationContainerInterface extends Singleton +interface ConfigurationContainerInterface { /** * Returns the option defined by section and key. From 30aa8ed10bebc4793cac722f5f9b8fd7241b56af Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 20 Mar 2018 11:02:43 +0100 Subject: [PATCH 61/75] TASK: Remove codacy from travis As we are no longer using codacy, we do not need to send any information. --- Makefile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Makefile b/Makefile index b1f4af0..21f6c31 100644 --- a/Makefile +++ b/Makefile @@ -41,16 +41,11 @@ unitTests: .Build/bin/phpunit --colors --debug -v \ -c Tests/Unit/UnitTests.xml -uploadCodeCoverage: uploadCodeCoverageToScrutinizer uploadCodeCoverageToCodacy +uploadCodeCoverage: uploadCodeCoverageToScrutinizer uploadCodeCoverageToScrutinizer: wget https://scrutinizer-ci.com/ocular.phar && \ php ocular.phar code-coverage:upload --format=php-clover .Build/report/functional/clover/coverage -uploadCodeCoverageToCodacy: - composer require -vv --dev codacy/coverage && \ - git checkout composer.json && \ - php .Build/bin/codacycoverage clover .Build/report/functional/clover/coverage - clean: rm -rf .Build composer.lock From 203b5d7adf4112273f55de57e9b877789811f69e Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Fri, 16 Mar 2018 17:37:19 +0100 Subject: [PATCH 62/75] TASK: Prevent issue When hook is called with non uid If some issues occur outside of our extension, we might not get a valid uid inside of our hooks. We will therefore add additional checks and prevent any further execution. Resolves: #112 --- Classes/Hook/DataHandler.php | 6 ++++- Tests/Unit/Hook/DataHandlerTest.php | 42 +++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/Classes/Hook/DataHandler.php b/Classes/Hook/DataHandler.php index e0b8e40..6c3da5c 100644 --- a/Classes/Hook/DataHandler.php +++ b/Classes/Hook/DataHandler.php @@ -72,7 +72,7 @@ class DataHandler implements Singleton /** * Called by CoreDataHandler on deletion of records. */ - public function processCmdmap_deleteAction(string $table, int $uid) : bool + public function processCmdmap_deleteAction(string $table, string $uid) : bool { if (! $this->shouldProcessHookForTable($table)) { $this->logger->debug('Delete not processed.', [$table, $uid]); @@ -95,6 +95,10 @@ class DataHandler implements Singleton $uid = $dataHandler->substNEWwithIDs[$uid]; } + if (!is_numeric($uid) || $uid <= 0) { + continue; + } + $this->processRecord($table, $uid); } } diff --git a/Tests/Unit/Hook/DataHandlerTest.php b/Tests/Unit/Hook/DataHandlerTest.php index ed68993..60da2e6 100644 --- a/Tests/Unit/Hook/DataHandlerTest.php +++ b/Tests/Unit/Hook/DataHandlerTest.php @@ -91,4 +91,46 @@ class DataHandlerToProcessorTest extends AbstractUnitTestCase ], ]; } + + /** + * @test + */ + public function indexingIsNotCalledForCacheClearIfDataIsInvalid() + { + $coreDataHandlerMock = $this->getMockBuilder(CoreDataHandler::class)->getMock(); + $ownDataHandlerMock = $this->getMockBuilder(OwnDataHandler::class) + ->disableOriginalConstructor() + ->getMock(); + + $subject = new DataHandler($ownDataHandlerMock); + + $ownDataHandlerMock->expects($this->never())->method('update'); + + $subject->clearCachePostProc([ + 'cacheCmd' => 'NEW343', + ], $coreDataHandlerMock); + } + /** + * @test + */ + public function indexingIsNotCalledForProcesIfDataIsInvalid() + { + $coreDataHandlerMock = $this->getMockBuilder(CoreDataHandler::class)->getMock(); + $coreDataHandlerMock->datamap = [ + 'tt_content' => [ + 'NEW343' => [], + ], + ]; + $coreDataHandlerMock->substNEWwithIDs = []; + + $ownDataHandlerMock = $this->getMockBuilder(OwnDataHandler::class) + ->disableOriginalConstructor() + ->getMock(); + + $subject = new DataHandler($ownDataHandlerMock); + + $ownDataHandlerMock->expects($this->never())->method('update'); + + $subject->processDatamap_afterAllOperations($coreDataHandlerMock); + } } From 5c493aad2117dbf9cf90b2d1c1c0cae9225d17b3 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 20 Mar 2018 14:18:47 +0100 Subject: [PATCH 63/75] TASK: Fix typo in test method name --- Tests/Unit/Hook/DataHandlerTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/Unit/Hook/DataHandlerTest.php b/Tests/Unit/Hook/DataHandlerTest.php index 60da2e6..aaf3935 100644 --- a/Tests/Unit/Hook/DataHandlerTest.php +++ b/Tests/Unit/Hook/DataHandlerTest.php @@ -113,7 +113,7 @@ class DataHandlerToProcessorTest extends AbstractUnitTestCase /** * @test */ - public function indexingIsNotCalledForProcesIfDataIsInvalid() + public function indexingIsNotCalledForProcessIfDataIsInvalid() { $coreDataHandlerMock = $this->getMockBuilder(CoreDataHandler::class)->getMock(); $coreDataHandlerMock->datamap = [ From 372cd484710173e50bd99f363d636cf97c4932a3 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 20 Mar 2018 15:42:34 +0100 Subject: [PATCH 64/75] FEATURE: Allow to copy a single field with CopyToProcessor --- Classes/DataProcessing/CopyToProcessor.php | 18 ++++++++++++++---- .../dataProcessing/CopyToProcessor.rst | 7 ++++++- .../DataProcessing/CopyToProcessorTest.php | 12 ++++++++++++ 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/Classes/DataProcessing/CopyToProcessor.php b/Classes/DataProcessing/CopyToProcessor.php index 28c4294..159701f 100644 --- a/Classes/DataProcessing/CopyToProcessor.php +++ b/Classes/DataProcessing/CopyToProcessor.php @@ -27,11 +27,21 @@ class CopyToProcessor implements ProcessorInterface { public function processData(array $record, array $configuration) : array { - $all = []; + $target = []; - $this->addArray($all, $record); - $all = array_filter($all); - $record[$configuration['to']] = implode(PHP_EOL, $all); + $from = $record; + if (isset($configuration['from'])) { + $from = $record[$configuration['from']]; + } + + if (is_array($from)) { + $this->addArray($target, $from); + } else { + $target[] = (string) $from; + } + + $target = array_filter($target); + $record[$configuration['to']] = implode(PHP_EOL, $target); return $record; } diff --git a/Documentation/source/configuration/dataProcessing/CopyToProcessor.rst b/Documentation/source/configuration/dataProcessing/CopyToProcessor.rst index 6d83d70..07faef1 100644 --- a/Documentation/source/configuration/dataProcessing/CopyToProcessor.rst +++ b/Documentation/source/configuration/dataProcessing/CopyToProcessor.rst @@ -8,6 +8,10 @@ Possible Options: ``to`` Defines the field to copy the values into. All values not false will be copied at the moment. +``from`` + Optional, defines the field to copy, can only be one field. + If empty, all existing fields will be copied. + Example:: plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing { @@ -17,7 +21,8 @@ Example:: } 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor 2 { - to = spellcheck + from = uid + to = backup_uid } } diff --git a/Tests/Unit/DataProcessing/CopyToProcessorTest.php b/Tests/Unit/DataProcessing/CopyToProcessorTest.php index 28545a3..6968d25 100644 --- a/Tests/Unit/DataProcessing/CopyToProcessorTest.php +++ b/Tests/Unit/DataProcessing/CopyToProcessorTest.php @@ -80,6 +80,18 @@ class CopyToProcessorTest extends AbstractUnitTestCase 'new_field' => 'Some content like lorem' . PHP_EOL . 'Tag 1' . PHP_EOL . 'Tag 2', ], ], + 'Copy single field to new field' => [ + 'record' => [ + 'field 1' => 'Some content like lorem', + ], + 'configuration' => [ + 'to' => 'new_field', + ], + 'expectedData' => [ + 'field 1' => 'Some content like lorem', + 'new_field' => 'Some content like lorem', + ], + ], ]; } } From c8dc95c4d37a831398b03f8d1f567fd6a2e583c4 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 20 Mar 2018 16:07:28 +0100 Subject: [PATCH 65/75] FEATURE: Provide type of result item In order to make further usage easier, we provide the type for each result item. This makes it possible to call {f:render(section: resultItem.type)}. --- .../Connection/Elasticsearch/SearchResult.php | 2 +- Classes/Connection/ResultItemInterface.php | 8 +++ Classes/Domain/Model/ResultItem.php | 13 +++- Classes/Domain/Model/SearchResult.php | 2 +- Classes/Domain/Search/SearchService.php | 11 ++-- Tests/Unit/Domain/Model/ResultItemTest.php | 59 +++++++++++++++++-- Tests/Unit/Domain/Model/SearchResultTest.php | 30 +++++++--- .../Unit/Domain/Search/SearchServiceTest.php | 17 +++++- 8 files changed, 119 insertions(+), 23 deletions(-) diff --git a/Classes/Connection/Elasticsearch/SearchResult.php b/Classes/Connection/Elasticsearch/SearchResult.php index 5b3d381..0f81463 100644 --- a/Classes/Connection/Elasticsearch/SearchResult.php +++ b/Classes/Connection/Elasticsearch/SearchResult.php @@ -108,7 +108,7 @@ class SearchResult implements SearchResultInterface } foreach ($this->result->getResults() as $result) { - $this->results[] = new ResultItem($result->getData()); + $this->results[] = new ResultItem($result->getData(), $result->getParam('_type')); } } diff --git a/Classes/Connection/ResultItemInterface.php b/Classes/Connection/ResultItemInterface.php index d7fe1a9..f5456c9 100644 --- a/Classes/Connection/ResultItemInterface.php +++ b/Classes/Connection/ResultItemInterface.php @@ -33,4 +33,12 @@ interface ResultItemInterface extends \ArrayAccess * Used e.g. for dataprocessing. */ public function getPlainData() : array; + + /** + * Returns the type of the item. + * + * That should make it easier to differentiate if multiple + * types are returned for one query. + */ + public function getType() : string; } diff --git a/Classes/Domain/Model/ResultItem.php b/Classes/Domain/Model/ResultItem.php index d0e370a..9ff45d2 100644 --- a/Classes/Domain/Model/ResultItem.php +++ b/Classes/Domain/Model/ResultItem.php @@ -29,9 +29,20 @@ class ResultItem implements ResultItemInterface */ protected $data = []; - public function __construct(array $result) + /** + * @var string + */ + protected $type = ''; + + public function __construct(array $result, string $type) { $this->data = $result; + $this->type = $type; + } + + public function getType() : string + { + return $this->type; } public function getPlainData() : array diff --git a/Classes/Domain/Model/SearchResult.php b/Classes/Domain/Model/SearchResult.php index 163b996..516c333 100644 --- a/Classes/Domain/Model/SearchResult.php +++ b/Classes/Domain/Model/SearchResult.php @@ -76,7 +76,7 @@ class SearchResult implements SearchResultInterface } foreach ($this->resultItems as $item) { - $this->results[] = new ResultItem($item); + $this->results[] = new ResultItem($item['data'], $item['type']); } } diff --git a/Classes/Domain/Search/SearchService.php b/Classes/Domain/Search/SearchService.php index 3a83b7b..1fc792e 100644 --- a/Classes/Domain/Search/SearchService.php +++ b/Classes/Domain/Search/SearchService.php @@ -146,10 +146,13 @@ class SearchService $newSearchResultItems = []; foreach ($this->configuration->get('searching.dataProcessing') as $configuration) { foreach ($searchResult as $resultItem) { - $newSearchResultItems[] = $this->dataProcessorService->executeDataProcessor( - $configuration, - $resultItem->getPlainData() - ); + $newSearchResultItems[] = [ + 'data' => $this->dataProcessorService->executeDataProcessor( + $configuration, + $resultItem->getPlainData() + ), + 'type' => $resultItem->getType(), + ]; } } diff --git a/Tests/Unit/Domain/Model/ResultItemTest.php b/Tests/Unit/Domain/Model/ResultItemTest.php index 53477c7..a4c18fd 100644 --- a/Tests/Unit/Domain/Model/ResultItemTest.php +++ b/Tests/Unit/Domain/Model/ResultItemTest.php @@ -36,7 +36,7 @@ class ResultItemTest extends AbstractUnitTestCase ]; $expectedData = $originalData; - $subject = new ResultItem($originalData); + $subject = new ResultItem($originalData, 'testType'); $this->assertSame( $expectedData, $subject->getPlainData(), @@ -55,7 +55,7 @@ class ResultItemTest extends AbstractUnitTestCase ]; $expectedData = $originalData; - $subject = new ResultItem($originalData); + $subject = new ResultItem($originalData, 'testType'); $this->assertSame( $originalData['title'], $subject['title'], @@ -73,7 +73,7 @@ class ResultItemTest extends AbstractUnitTestCase 'title' => 'Some title', ]; - $subject = new ResultItem($originalData); + $subject = new ResultItem($originalData, 'testType'); $this->assertTrue(isset($subject['title']), 'Could not determine that title exists.'); $this->assertFalse(isset($subject['title2']), 'Could not determine that title2 does not exists.'); } @@ -88,7 +88,7 @@ class ResultItemTest extends AbstractUnitTestCase 'title' => 'Some title', ]; - $subject = new ResultItem($originalData); + $subject = new ResultItem($originalData, 'testType'); $this->expectException(\BadMethodCallException::class); $subject['title'] = 'New Title'; } @@ -103,8 +103,57 @@ class ResultItemTest extends AbstractUnitTestCase 'title' => 'Some title', ]; - $subject = new ResultItem($originalData); + $subject = new ResultItem($originalData, 'testType'); $this->expectException(\BadMethodCallException::class); unset($subject['title']); } + + /** + * @test + */ + public function typeCanBeRetrievedAfterConstruction() + { + $originalData = [ + 'uid' => 10, + 'title' => 'Some title', + ]; + $expectedData = $originalData; + + $subject = new ResultItem($originalData, 'testType'); + $this->assertSame( + 'testType', + $subject->getType(), + 'Could not retrieve type.' + ); + } + + /** + * @test + */ + public function typeCanNotBeChanged() + { + $originalData = [ + 'uid' => 10, + 'title' => 'Some title', + ]; + + $subject = new ResultItem($originalData, 'testType'); + $this->expectException(\BadMethodCallException::class); + $subject['type'] = 'New Title'; + } + + /** + * @test + */ + public function typeCanNotBeRemoved() + { + $originalData = [ + 'uid' => 10, + 'title' => 'Some title', + ]; + + $subject = new ResultItem($originalData, 'testType'); + $this->expectException(\BadMethodCallException::class); + unset($subject['type']); + } } diff --git a/Tests/Unit/Domain/Model/SearchResultTest.php b/Tests/Unit/Domain/Model/SearchResultTest.php index eebb3c1..db5ccb5 100644 --- a/Tests/Unit/Domain/Model/SearchResultTest.php +++ b/Tests/Unit/Domain/Model/SearchResultTest.php @@ -71,16 +71,25 @@ class SearchResultTest extends AbstractUnitTestCase $originalSearchResultMock = $this->getMockBuilder(SearchResultInterface::class)->getMock(); $data = [ [ - 'uid' => 10, - 'title' => 'Some Title', + 'data' => [ + 'uid' => 10, + 'title' => 'Some Title', + ], + 'type' => 'testType1', ], [ - 'uid' => 11, - 'title' => 'Some Title 2', + 'data' => [ + 'uid' => 11, + 'title' => 'Some Title 2', + ], + 'type' => 'testType2', ], [ - 'uid' => 12, - 'title' => 'Some Title 3', + 'data' => [ + 'uid' => 12, + 'title' => 'Some Title 3', + ], + 'type' => 'testType2', ], ]; @@ -89,9 +98,12 @@ class SearchResultTest extends AbstractUnitTestCase $this->assertCount(3, $resultItems); - $this->assertSame($data[0]['uid'], $resultItems[0]['uid']); - $this->assertSame($data[1]['uid'], $resultItems[1]['uid']); - $this->assertSame($data[2]['uid'], $resultItems[2]['uid']); + $this->assertSame($data[0]['data']['uid'], $resultItems[0]['uid']); + $this->assertSame($data[0]['type'], $resultItems[0]->getType()); + $this->assertSame($data[1]['data']['uid'], $resultItems[1]['uid']); + $this->assertSame($data[1]['type'], $resultItems[1]->getType()); + $this->assertSame($data[2]['data']['uid'], $resultItems[2]['uid']); + $this->assertSame($data[2]['type'], $resultItems[2]->getType()); $this->assertInstanceOf(ResultItemInterface::class, $resultItems[0]); $this->assertInstanceOf(ResultItemInterface::class, $resultItems[1]); diff --git a/Tests/Unit/Domain/Search/SearchServiceTest.php b/Tests/Unit/Domain/Search/SearchServiceTest.php index 3d6791e..50c1f49 100644 --- a/Tests/Unit/Domain/Search/SearchServiceTest.php +++ b/Tests/Unit/Domain/Search/SearchServiceTest.php @@ -294,7 +294,14 @@ class SearchServiceTest extends AbstractUnitTestCase )); $searchResultMock = $this->getMockBuilder(SearchResultInterface::class)->getMock(); - $searchResult = new SearchResult($searchResultMock, [['field 1' => 'value 1']]); + $searchResult = new SearchResult($searchResultMock, [ + [ + 'data' => [ + 'field 1' => 'value 1' + ], + 'type' => 'testType', + ], + ]); $this->connection->expects($this->once()) ->method('search') @@ -311,7 +318,13 @@ class SearchServiceTest extends AbstractUnitTestCase $this->objectManager->expects($this->once()) ->method('get') ->with(SearchResult::class, $searchResult, [ - ['field 1' => 'value 1', 'field 2' => 'value 2'] + [ + 'data' => [ + 'field 1' => 'value 1', + 'field 2' => 'value 2', + ], + 'type' => 'testType', + ] ]) ->willReturn($searchResultMock); From f9da96e7df96132182476cbf87f0e96b00692bf9 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 22 Mar 2018 13:43:25 +0100 Subject: [PATCH 66/75] TASK: Raise release version --- Documentation/source/conf.py | 6 +++--- Documentation/source/installation.rst | 2 +- ext_emconf.php | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Documentation/source/conf.py b/Documentation/source/conf.py index aa846e0..92dc768 100644 --- a/Documentation/source/conf.py +++ b/Documentation/source/conf.py @@ -59,9 +59,9 @@ author = u'Daniel Siepmann' # built documents. # # The short X.Y version. -version = u'0.0.1' +version = u'0.0.3' # The full version, including alpha/beta/rc tags. -release = u'0.0.1' +release = u'0.0.3' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -133,7 +133,7 @@ html_theme_options = { # The name for this set of Sphinx documents. # " v documentation" by default. -#html_title = u'TYPO3 Extension search_core v0.0.1' +#html_title = u'TYPO3 Extension search_core v0.0.3' # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None diff --git a/Documentation/source/installation.rst b/Documentation/source/installation.rst index 0a0e5b6..174d9f7 100644 --- a/Documentation/source/installation.rst +++ b/Documentation/source/installation.rst @@ -9,7 +9,7 @@ Composer The extension can be installed through composer:: - composer require "codappix/search_core" "~1.0.0" + composer require "codappix/search_core" "~0.0.3" Note that you have to allow unstable packages: diff --git a/ext_emconf.php b/ext_emconf.php index 7c3a087..e2b4d1b 100644 --- a/ext_emconf.php +++ b/ext_emconf.php @@ -18,7 +18,7 @@ $EM_CONF[$_EXTKEY] = [ ], ], 'state' => 'beta', - 'version' => '0.0.1', + 'version' => '0.0.3', 'author' => 'Daniel Siepmann', 'author_email' => 'coding@daniel-siepmann.de', ]; From 7cb160e5498e89c04dadb8ff789df0a0af0d346d Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 10 Apr 2018 17:17:16 +0200 Subject: [PATCH 67/75] TASK: Fix some small errors in changelog Fix wrong month used in changelog. Fix typos in headlines of changelog. Keep same case for features. --- Documentation/source/changelog.rst | 10 +++++----- ...ration.rst => 20180306-120-facet-configuration.rst} | 4 ++-- ...r.rst => 20180308-131-respect-page-cache-clear.rst} | 0 ...nts.rst => 20180308-introduce-php70-type-hints.rst} | 4 ++-- ...id.rst => 20180309-25-provide-sys-language-uid.rst} | 2 +- ...> 20180315-134-make-conent-fields-configurable.rst} | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) rename Documentation/source/changelog/{20180406-120-facet-configuration.rst => 20180306-120-facet-configuration.rst} (93%) rename Documentation/source/changelog/{20180408-131-respect-page-cache-clear.rst => 20180308-131-respect-page-cache-clear.rst} (100%) rename Documentation/source/changelog/{20180408-introduce-php70-type-hints.rst => 20180308-introduce-php70-type-hints.rst} (82%) rename Documentation/source/changelog/{20180409-25-provide-sys-language-uid.rst => 20180309-25-provide-sys-language-uid.rst} (91%) rename Documentation/source/changelog/{20180415-134-make-conent-fields-configurable.rst => 20180315-134-make-conent-fields-configurable.rst} (88%) diff --git a/Documentation/source/changelog.rst b/Documentation/source/changelog.rst index 8b1c649..61756c6 100644 --- a/Documentation/source/changelog.rst +++ b/Documentation/source/changelog.rst @@ -5,8 +5,8 @@ Changelog :maxdepth: 1 :glob: - changelog/20180415-134-make-conent-fields-configurable - changelog/20180409-25-provide-sys-language-uid - changelog/20180408-131-respect-page-cache-clear - changelog/20180408-introduce-php70-type-hints - changelog/20180406-120-facet-configuration + changelog/20180315-134-make-conent-fields-configurable + changelog/20180309-25-provide-sys-language-uid + changelog/20180308-131-respect-page-cache-clear + changelog/20180308-introduce-php70-type-hints + changelog/20180306-120-facet-configuration diff --git a/Documentation/source/changelog/20180406-120-facet-configuration.rst b/Documentation/source/changelog/20180306-120-facet-configuration.rst similarity index 93% rename from Documentation/source/changelog/20180406-120-facet-configuration.rst rename to Documentation/source/changelog/20180306-120-facet-configuration.rst index 0c2338f..88c36d0 100644 --- a/Documentation/source/changelog/20180406-120-facet-configuration.rst +++ b/Documentation/source/changelog/20180306-120-facet-configuration.rst @@ -1,5 +1,5 @@ -Breacking Change 120 "Pass facets configuration to Elasticsearch" -================================================================= +Breaking Change 120 "Pass facets configuration to Elasticsearch" +================================================================ In order to allow arbitrary facet configuration, we do not process the facet configuration anymore. Instead integrators are able to configure facets for search service "as is". We just pipe the diff --git a/Documentation/source/changelog/20180408-131-respect-page-cache-clear.rst b/Documentation/source/changelog/20180308-131-respect-page-cache-clear.rst similarity index 100% rename from Documentation/source/changelog/20180408-131-respect-page-cache-clear.rst rename to Documentation/source/changelog/20180308-131-respect-page-cache-clear.rst diff --git a/Documentation/source/changelog/20180408-introduce-php70-type-hints.rst b/Documentation/source/changelog/20180308-introduce-php70-type-hints.rst similarity index 82% rename from Documentation/source/changelog/20180408-introduce-php70-type-hints.rst rename to Documentation/source/changelog/20180308-introduce-php70-type-hints.rst index cc2dfe3..dae8b40 100644 --- a/Documentation/source/changelog/20180408-introduce-php70-type-hints.rst +++ b/Documentation/source/changelog/20180308-introduce-php70-type-hints.rst @@ -1,5 +1,5 @@ -Breacking Change "Introduce PHP 7.0 TypeHints" -============================================== +Breaking Change "Introduce PHP 7.0 TypeHints" +============================================= As PHP evolved, we now migrate the whole code base to use PHP 7.0 type hints. We do not use PHP 7.1 Type Hints, as some customers still need PHP 7.0 support. diff --git a/Documentation/source/changelog/20180409-25-provide-sys-language-uid.rst b/Documentation/source/changelog/20180309-25-provide-sys-language-uid.rst similarity index 91% rename from Documentation/source/changelog/20180409-25-provide-sys-language-uid.rst rename to Documentation/source/changelog/20180309-25-provide-sys-language-uid.rst index 3700ce0..5230b75 100644 --- a/Documentation/source/changelog/20180409-25-provide-sys-language-uid.rst +++ b/Documentation/source/changelog/20180309-25-provide-sys-language-uid.rst @@ -1,4 +1,4 @@ -FEATURE 25 "Respect multiple languages" - Provide sys_language_uid +Feature 25 "Respect multiple languages" - Provide sys_language_uid ================================================================== Previously we did not fetch ``sys_language_uid`` field from database. This prevented everyone from diff --git a/Documentation/source/changelog/20180415-134-make-conent-fields-configurable.rst b/Documentation/source/changelog/20180315-134-make-conent-fields-configurable.rst similarity index 88% rename from Documentation/source/changelog/20180415-134-make-conent-fields-configurable.rst rename to Documentation/source/changelog/20180315-134-make-conent-fields-configurable.rst index 7068119..2601a71 100644 --- a/Documentation/source/changelog/20180415-134-make-conent-fields-configurable.rst +++ b/Documentation/source/changelog/20180315-134-make-conent-fields-configurable.rst @@ -1,4 +1,4 @@ -FEATURE 134 "Enable indexing of tt_content records of CType Header" +Feature 134 "Enable indexing of tt_content records of CType Header" =================================================================== Before, only ``bodytext`` was used to generate content while indexing pages. From c5da12f854d769c9d1ca0bc3debb14bfde5fca06 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 12 Apr 2018 08:27:41 +0200 Subject: [PATCH 68/75] TASK: Make travis pass elasticsearch start step Travis has installed elasticsearch already, we therefore need to REstart instead of start, as starting again will not work. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 56f42fa..54c2637 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ addons: packages: - oracle-java8-set-default before_install: - - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.2.0.deb && sudo dpkg -i --force-confnew elasticsearch-5.2.0.deb && sudo service elasticsearch start + - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.2.0.deb && sudo dpkg -i --force-confnew elasticsearch-5.2.0.deb && sudo service elasticsearch restart - mysql -u root -e 'GRANT ALL ON `typo3_ci_ft%`.* TO travis@127.0.0.1;' language: php From 97c0485e6f85c1a31acec74f3d60160ea4dcaf33 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 12 Apr 2018 08:27:41 +0200 Subject: [PATCH 69/75] TASK: Make travis pass elasticsearch start step Travis has installed elasticsearch already, we therefore need to Restart instead of start, as starting again will not work. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 56f42fa..54c2637 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ addons: packages: - oracle-java8-set-default before_install: - - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.2.0.deb && sudo dpkg -i --force-confnew elasticsearch-5.2.0.deb && sudo service elasticsearch start + - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.2.0.deb && sudo dpkg -i --force-confnew elasticsearch-5.2.0.deb && sudo service elasticsearch restart - mysql -u root -e 'GRANT ALL ON `typo3_ci_ft%`.* TO travis@127.0.0.1;' language: php From 4234575c36dbfbf1bf13fac0c05aa8a26280398c Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 12 Apr 2018 08:26:34 +0200 Subject: [PATCH 70/75] TASK: Fix broken tests caused by missing caches For some reason, which is unknown yet, caches are needed to make tests pass. We therefore configure the caches to be null. --- Tests/Unit/AbstractUnitTestCase.php | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/Tests/Unit/AbstractUnitTestCase.php b/Tests/Unit/AbstractUnitTestCase.php index fa29d57..384236c 100644 --- a/Tests/Unit/AbstractUnitTestCase.php +++ b/Tests/Unit/AbstractUnitTestCase.php @@ -39,14 +39,9 @@ abstract class AbstractUnitTestCase extends CoreTestCase $this->singletonInstances = GeneralUtility::getSingletonInstances(); // Disable caching backends to make TYPO3 parts work in unit test mode. - \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance( \TYPO3\CMS\Core\Cache\CacheManager::class - )->setCacheConfigurations([ - 'extbase_object' => [ - 'backend' => \TYPO3\CMS\Core\Cache\Backend\NullBackend::class, - ], - ]); + )->setCacheConfigurations($this->getCacheConfiguration()); } public function tearDown() @@ -100,4 +95,25 @@ abstract class AbstractUnitTestCase extends CoreTestCase { return \TYPO3\CMS\Core\Utility\VersionNumberUtility::convertVersionNumberToInteger(TYPO3_version) < 8000000; } + + protected function getCacheConfiguration() : array + { + $cacheConfiguration = [ + 'extbase_object' => [ + 'backend' => \TYPO3\CMS\Core\Cache\Backend\NullBackend::class, + ], + 'cache_runtime' => [ + 'backend' => \TYPO3\CMS\Core\Cache\Backend\NullBackend::class, + ], + ]; + + if (class_exists(\TYPO3\CMS\Fluid\Core\Cache\FluidTemplateCache::class)) { + $cacheConfiguration['fluid_template'] = [ + 'backend' => \TYPO3\CMS\Core\Cache\Backend\NullBackend::class, + 'frontend' => \TYPO3\CMS\Fluid\Core\Cache\FluidTemplateCache::class, + ]; + } + + return $cacheConfiguration; + } } From c1d1e6ed68db911a1935d1b84a3e781b4188a223 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 1 Apr 2018 11:29:22 +0200 Subject: [PATCH 71/75] BUGFIX: Do not resolve inputDateTime as relation Add test which checks this behaviour. Exclude inputDateTime from relation resolving. Resolves #147 --- .../Index/TcaIndexer/RelationResolver.php | 2 +- .../Index/TcaIndexer/RelationResolverTest.php | 83 +++++++++++++++++++ 2 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 Tests/Unit/Domain/Index/TcaIndexer/RelationResolverTest.php diff --git a/Classes/Domain/Index/TcaIndexer/RelationResolver.php b/Classes/Domain/Index/TcaIndexer/RelationResolver.php index ae47e30..763589b 100644 --- a/Classes/Domain/Index/TcaIndexer/RelationResolver.php +++ b/Classes/Domain/Index/TcaIndexer/RelationResolver.php @@ -82,7 +82,7 @@ class RelationResolver implements Singleton protected function isRelation(array &$config) : bool { return isset($config['foreign_table']) - || (isset($config['renderType']) && $config['renderType'] !== 'selectSingle') + || (isset($config['renderType']) && !in_array($config['renderType'], ['selectSingle', 'inputDateTime'])) || (isset($config['internal_type']) && strtolower($config['internal_type']) === 'db') ; } diff --git a/Tests/Unit/Domain/Index/TcaIndexer/RelationResolverTest.php b/Tests/Unit/Domain/Index/TcaIndexer/RelationResolverTest.php new file mode 100644 index 0000000..c227c24 --- /dev/null +++ b/Tests/Unit/Domain/Index/TcaIndexer/RelationResolverTest.php @@ -0,0 +1,83 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Domain\Index\TcaIndexer\RelationResolver; +use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class RelationResolverTest extends AbstractUnitTestCase +{ + /** + * @var RelationResolver + */ + protected $subject; + + public function setUp() + { + parent::setUp(); + $this->subject = new RelationResolver(); + } + + /** + * @test + */ + public function renderTypeInputDateTimeIsHandled() + { + $originalRecord = [ + 'starttime' => 0, + ]; + $record = $originalRecord; + $GLOBALS['TCA'] = [ + 'tt_content' => [ + 'columns' => [ + 'starttime' => [ + 'config' => [ + 'default' => 0, + 'eval' => 'datetime', + 'renderType' => 'inputDateTime', + ], + 'type' => 'input', + 'exclude' => 1, + 'l10n_display' => 'defaultAsReadonly', + 'l10n_mode' => 'exclude', + 'label' => 'LLL:EXT:lang/Resources/Private/Language/locallang_general.xlf:LGL.starttime', + ], + ], + ], + ]; + $tableServiceMock = $this->getMockBuilder(TcaTableServiceInterface::class)->getMock(); + $tableServiceMock->expects($this->any()) + ->method('getTableName') + ->willReturn('tt_content'); + $tableServiceMock->expects($this->any()) + ->method('getColumnConfig') + ->willReturn($GLOBALS['TCA']['tt_content']['columns']['starttime']['config']); + + $this->subject->resolveRelationsForRecord($tableServiceMock, $record); + + $this->assertSame( + $originalRecord, + $record, + 'TCA column configured with renderType inputDateTime was not kept as unix timestamp.' + ); + } +} From 82d397c4287af728c8460ba53495fb328f4a421f Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 1 Apr 2018 11:13:23 +0200 Subject: [PATCH 72/75] FEATURE: Keep sys_language_uid while indexing Before this change, sys_language_uid was indexed as an empty string, due to internal used TYPO3 API. We now skip the configure field which defines language uid. This way you can build filter based on current language. Resolves #148 --- .../Index/TcaIndexer/RelationResolver.php | 3 +- .../Index/TcaIndexer/TcaTableService.php | 9 ++ .../Index/TcaIndexer/TcaTableService76.php | 9 ++ .../TcaIndexer/TcaTableServiceInterface.php | 2 + Documentation/source/changelog.rst | 1 + .../20180410-148-keep-sys_language_uid.rst | 10 ++ Tests/Functional/Indexing/TcaIndexerTest.php | 6 +- .../Index/TcaIndexer/RelationResolverTest.php | 92 +++++++++++++++++++ 8 files changed, 126 insertions(+), 6 deletions(-) create mode 100644 Documentation/source/changelog/20180410-148-keep-sys_language_uid.rst create mode 100644 Tests/Unit/Domain/Index/TcaIndexer/RelationResolverTest.php diff --git a/Classes/Domain/Index/TcaIndexer/RelationResolver.php b/Classes/Domain/Index/TcaIndexer/RelationResolver.php index ae47e30..378dc75 100644 --- a/Classes/Domain/Index/TcaIndexer/RelationResolver.php +++ b/Classes/Domain/Index/TcaIndexer/RelationResolver.php @@ -37,7 +37,8 @@ class RelationResolver implements Singleton { foreach (array_keys($record) as $column) { // TODO: Define / configure fields to exclude?! - if ($column === 'pid') { + if (in_array($column, ['pid', $service->getLanguageUidColumn()])) { + $record[$column] = (int) $record[$column]; continue; } diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index ee0a2d8..37b0c89 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -283,6 +283,15 @@ class TcaTableService implements TcaTableServiceInterface return $this->tca['columns'][$columnName]['config']; } + public function getLanguageUidColumn() : string + { + if (!isset($this->tca['ctrl']['languageField'])) { + return ''; + } + + return $this->tca['ctrl']['languageField']; + } + /** * Checks whether the given record was blacklisted by root line. * This can be configured by typoscript as whole root lines can be black listed. diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService76.php b/Classes/Domain/Index/TcaIndexer/TcaTableService76.php index f52027a..59d6136 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService76.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService76.php @@ -290,6 +290,15 @@ class TcaTableService76 implements TcaTableServiceInterface return $this->tca['columns'][$columnName]['config']; } + public function getLanguageUidColumn() : string + { + if (!isset($this->tca['ctrl']['languageField'])) { + return ''; + } + + return $this->tca['ctrl']['languageField']; + } + /** * Checks whether the given record was blacklisted by root line. * This can be configured by typoscript as whole root lines can be black listed. diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableServiceInterface.php b/Classes/Domain/Index/TcaIndexer/TcaTableServiceInterface.php index 0c9cfc4..e4ee7c5 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableServiceInterface.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableServiceInterface.php @@ -41,4 +41,6 @@ interface TcaTableServiceInterface public function getRecords(int $offset, int $limit) : array; public function getRecord(int $identifier) : array; + + public function getLanguageUidColumn() : string; } diff --git a/Documentation/source/changelog.rst b/Documentation/source/changelog.rst index 61756c6..37b42c6 100644 --- a/Documentation/source/changelog.rst +++ b/Documentation/source/changelog.rst @@ -5,6 +5,7 @@ Changelog :maxdepth: 1 :glob: + changelog/20180410-148-keep-sys_language_uid changelog/20180315-134-make-conent-fields-configurable changelog/20180309-25-provide-sys-language-uid changelog/20180308-131-respect-page-cache-clear diff --git a/Documentation/source/changelog/20180410-148-keep-sys_language_uid.rst b/Documentation/source/changelog/20180410-148-keep-sys_language_uid.rst new file mode 100644 index 0000000..d89a289 --- /dev/null +++ b/Documentation/source/changelog/20180410-148-keep-sys_language_uid.rst @@ -0,0 +1,10 @@ +Feature 148 "Cast sys_language_uid to int" +========================================== + +While resolving relations the configured language uid field, fetched from TCA, will +be casted to integer and returned immediately. + +This change prevents the bug mentioned in :issue:`148`, where `0` is casted to an +empty string, which makes filtering hard. + +See :issue:`148`. diff --git a/Tests/Functional/Indexing/TcaIndexerTest.php b/Tests/Functional/Indexing/TcaIndexerTest.php index d7ecc3a..85f36a6 100644 --- a/Tests/Functional/Indexing/TcaIndexerTest.php +++ b/Tests/Functional/Indexing/TcaIndexerTest.php @@ -103,11 +103,7 @@ class TcaIndexerTest extends AbstractFunctionalTestCase ->with( $this->stringContains('tt_content'), $this->callback(function ($documents) { - if ($this->isLegacyVersion()) { - return isset($documents[0]['sys_language_uid']) && $documents[0]['sys_language_uid'] === '2'; - } else { - return isset($documents[0]['sys_language_uid']) && $documents[0]['sys_language_uid'] === 2; - } + return isset($documents[0]['sys_language_uid']) && $documents[0]['sys_language_uid'] === 2; }) ); diff --git a/Tests/Unit/Domain/Index/TcaIndexer/RelationResolverTest.php b/Tests/Unit/Domain/Index/TcaIndexer/RelationResolverTest.php new file mode 100644 index 0000000..60732dd --- /dev/null +++ b/Tests/Unit/Domain/Index/TcaIndexer/RelationResolverTest.php @@ -0,0 +1,92 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Domain\Index\TcaIndexer\RelationResolver; +use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class RelationResolverTest extends AbstractUnitTestCase +{ + /** + * @var RelationResolver + */ + protected $subject; + + public function setUp() + { + parent::setUp(); + $this->subject = new RelationResolver(); + } + + /** + * @test + */ + public function sysLanguageUidZeroIsKept() + { + $record = [ + 'sys_language_uid' => '0', + ]; + $GLOBALS['TCA'] = [ + 'tt_content' => [ + 'columns' => [ + 'sys_language_uid' => [ + 'config' => [ + 'default' => 0, + 'items' => [ + [ + 'LLL:EXT:lang/Resources/Private/Language/locallang_general.xlf:LGL.allLanguages', + '-1', + 'flags-multiple', + ], + ], + 'renderType' => 'selectSingle', + 'special' => 'languages', + 'type' => 'select', + 'exclude' => '1', + 'label' => 'LLL:EXT:lang/Resources/Private/Language/locallang_general.xlf:LGL.language', + ], + ], + ], + ], + ]; + $tableServiceMock = $this->getMockBuilder(TcaTableServiceInterface::class)->getMock(); + $tableServiceMock->expects($this->any()) + ->method('getTableName') + ->willReturn('tt_content'); + $tableServiceMock->expects($this->any()) + ->method('getLanguageUidColumn') + ->willReturn('sys_language_uid'); + $tableServiceMock->expects($this->any()) + ->method('getColumnConfig') + ->willReturn($GLOBALS['TCA']['tt_content']['columns']['sys_language_uid']['config']); + + $this->subject->resolveRelationsForRecord($tableServiceMock, $record); + + $this->assertSame( + [ + 'sys_language_uid' => 0, + ], + $record, + 'sys_language_uid was not kept as zero.' + ); + } +} From dfb3f897e7e5e30c8ebba0d3dd0b69b1d14dcd7f Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 24 Apr 2018 09:37:25 +0200 Subject: [PATCH 73/75] !!!|FEATURE: Extract relation resolver to data processor Instead of hardcoding and tight coupling of relation resolving, we now provide a dataprocessor instead. Therefore you need to configure resolving for each indexed type. This resolves #149 and #147. --- Classes/DataProcessing/Service.php | 6 +- .../TcaRelationResolvingProcessor.php | 101 +++++++++++++++++ Classes/Domain/Index/AbstractIndexer.php | 2 +- .../Domain/Index/TcaIndexer/PagesIndexer.php | 3 +- .../Index/TcaIndexer/RelationResolver.php | 5 +- .../Index/TcaIndexer/TcaTableService.php | 9 -- .../Index/TcaIndexer/TcaTableService76.php | 9 -- Documentation/source/changelog.rst | 1 + ...t-relation-resolver-to-data-processing.rst | 24 ++++ .../TcaRelationResolvingProcessor.rst | 34 ++++++ Documentation/source/dataprocessors.rst | 16 +-- .../TcaRelationResolvingProcessorTest.php} | 26 ++--- Tests/Functional/Fixtures/BasicSetup.ts | 8 ++ Tests/Functional/Indexing/TcaIndexerTest.php | 32 ------ Tests/Unit/Bootstrap.php | 2 + .../TcaRelationResolvingProcessorTest.php} | 105 ++++++++++++------ 16 files changed, 262 insertions(+), 121 deletions(-) create mode 100644 Classes/DataProcessing/TcaRelationResolvingProcessor.php create mode 100644 Documentation/source/changelog/20180424-149-extract-relation-resolver-to-data-processing.rst create mode 100644 Documentation/source/configuration/dataProcessing/TcaRelationResolvingProcessor.rst rename Tests/Functional/{Indexing/TcaIndexer/RelationResolverTest.php => DataProcessing/TcaRelationResolvingProcessorTest.php} (77%) rename Tests/Unit/{Domain/Index/TcaIndexer/RelationResolverTest.php => DataProcessing/TcaRelationResolvingProcessorTest.php} (53%) diff --git a/Classes/DataProcessing/Service.php b/Classes/DataProcessing/Service.php index 3e3b053..5369b86 100644 --- a/Classes/DataProcessing/Service.php +++ b/Classes/DataProcessing/Service.php @@ -42,7 +42,7 @@ class Service * * @param array|string $configuration Either the full configuration or only the class name. */ - public function executeDataProcessor($configuration, array $data) : array + public function executeDataProcessor($configuration, array $data, string $recordType = '') : array { if (is_string($configuration)) { $configuration = [ @@ -50,6 +50,10 @@ class Service ]; } + if (!isset($configuration['_table']) && $recordType !== '') { + $configuration['_table'] = $recordType; + } + return $this->objectManager->get($configuration['_typoScriptNodeValue']) ->processData($data, $configuration); } diff --git a/Classes/DataProcessing/TcaRelationResolvingProcessor.php b/Classes/DataProcessing/TcaRelationResolvingProcessor.php new file mode 100644 index 0000000..e111d7d --- /dev/null +++ b/Classes/DataProcessing/TcaRelationResolvingProcessor.php @@ -0,0 +1,101 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Domain\Index\TcaIndexer\RelationResolver; +use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface; +use TYPO3\CMS\Core\Utility\GeneralUtility; +use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; + +/** + * Resolves relations from TCA using RelationResolver. + */ +class TcaRelationResolvingProcessor implements ProcessorInterface +{ + /** + * @var ObjectManagerInterface + */ + protected $objectManager; + + /** + * @var RelationResolver + */ + protected $relationResolver; + + public function __construct( + ObjectManagerInterface $objectManager, + RelationResolver $relationResolver + ) { + $this->objectManager = $objectManager; + $this->relationResolver = $relationResolver; + } + + /** + * @throws \InvalidArgumentException If _table is not configured. + */ + public function processData(array $record, array $configuration) : array + { + $this->initializeConfiguration($configuration); + + $tcaTableService = $this->objectManager->get( + TcaTableServiceInterface::class, + $configuration['_table'] + ); + + $processedRecord = $this->relationResolver->resolveRelationsForRecord( + $tcaTableService, + $this->getRecordToProcess($record, $configuration) + ); + + return array_merge($record, $processedRecord); + } + + /** + * @throws \InvalidArgumentException If _table is not configured. + */ + protected function initializeConfiguration(array &$configuration) + { + if (!isset($configuration['_table'])) { + throw new \InvalidArgumentException('The configuration "_table" is mandantory.', 1524552631); + } + + if (!isset($configuration['excludeFields'])) { + $configuration['excludeFields'] = ''; + } + + $configuration['excludeFields'] = GeneralUtility::trimExplode(',', $configuration['excludeFields'], true); + } + + protected function getRecordToProcess(array $record, array $configuration) : array + { + if ($configuration['excludeFields'] === []) { + return $record; + } + + $newRecord = []; + $keysToUse = array_diff(array_keys($record), $configuration['excludeFields']); + foreach ($keysToUse as $keyToUse) { + $newRecord[$keyToUse] = $record[$keyToUse]; + } + + return $newRecord; + } +} diff --git a/Classes/Domain/Index/AbstractIndexer.php b/Classes/Domain/Index/AbstractIndexer.php index 6644716..b22c011 100644 --- a/Classes/Domain/Index/AbstractIndexer.php +++ b/Classes/Domain/Index/AbstractIndexer.php @@ -129,7 +129,7 @@ abstract class AbstractIndexer implements IndexerInterface { try { foreach ($this->configuration->get('indexing.' . $this->identifier . '.dataProcessing') as $configuration) { - $record = $this->dataProcessorService->executeDataProcessor($configuration, $record); + $record = $this->dataProcessorService->executeDataProcessor($configuration, $record, $this->identifier); } } catch (InvalidArgumentException $e) { // Nothing to do. diff --git a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php index 9cd4764..af1d129 100644 --- a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php +++ b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php @@ -60,6 +60,8 @@ class PagesIndexer extends TcaIndexer protected function prepareRecord(array &$record) { + parent::prepareRecord($record); + $possibleTitleFields = ['nav_title', 'tx_tqseo_pagetitle_rel', 'title']; foreach ($possibleTitleFields as $searchTitleField) { if (isset($record[$searchTitleField]) && trim($record[$searchTitleField])) { @@ -74,7 +76,6 @@ class PagesIndexer extends TcaIndexer $record['content'] = $content['content']; $record['media'] = array_values(array_unique(array_merge($record['media'], $content['images']))); } - parent::prepareRecord($record); } protected function fetchContentForPage(int $uid) : array diff --git a/Classes/Domain/Index/TcaIndexer/RelationResolver.php b/Classes/Domain/Index/TcaIndexer/RelationResolver.php index 256a135..ae10f68 100644 --- a/Classes/Domain/Index/TcaIndexer/RelationResolver.php +++ b/Classes/Domain/Index/TcaIndexer/RelationResolver.php @@ -33,10 +33,9 @@ use TYPO3\CMS\Core\Utility\GeneralUtility; */ class RelationResolver implements Singleton { - public function resolveRelationsForRecord(TcaTableServiceInterface $service, array &$record) + public function resolveRelationsForRecord(TcaTableServiceInterface $service, array $record) : array { foreach (array_keys($record) as $column) { - // TODO: Define / configure fields to exclude?! if (in_array($column, ['pid', $service->getLanguageUidColumn()])) { $record[$column] = (int) $record[$column]; continue; @@ -62,6 +61,8 @@ class RelationResolver implements Singleton continue; } } + + return $record; } protected function resolveValue($value, array $tcaColumn) diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index 37b0c89..1330e22 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -54,11 +54,6 @@ class TcaTableService implements TcaTableServiceInterface */ protected $configuration; - /** - * @var RelationResolver - */ - protected $relationResolver; - /** * @var \TYPO3\CMS\Core\Log\Logger */ @@ -93,7 +88,6 @@ class TcaTableService implements TcaTableServiceInterface */ public function __construct( $tableName, - RelationResolver $relationResolver, ConfigurationContainerInterface $configuration ) { if (!isset($GLOBALS['TCA'][$tableName])) { @@ -106,7 +100,6 @@ class TcaTableService implements TcaTableServiceInterface $this->tableName = $tableName; $this->tca = &$GLOBALS['TCA'][$this->tableName]; $this->configuration = $configuration; - $this->relationResolver = $relationResolver; } public function getTableName() : string @@ -151,8 +144,6 @@ class TcaTableService implements TcaTableServiceInterface public function prepareRecord(array &$record) { - $this->relationResolver->resolveRelationsForRecord($this, $record); - if (isset($record['uid']) && !isset($record['search_identifier'])) { $record['search_identifier'] = $record['uid']; } diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService76.php b/Classes/Domain/Index/TcaIndexer/TcaTableService76.php index 59d6136..4445d6d 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService76.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService76.php @@ -50,11 +50,6 @@ class TcaTableService76 implements TcaTableServiceInterface */ protected $configuration; - /** - * @var RelationResolver - */ - protected $relationResolver; - /** * @var \TYPO3\CMS\Core\Log\Logger */ @@ -89,7 +84,6 @@ class TcaTableService76 implements TcaTableServiceInterface */ public function __construct( $tableName, - RelationResolver $relationResolver, ConfigurationContainerInterface $configuration ) { if (!isset($GLOBALS['TCA'][$tableName])) { @@ -102,7 +96,6 @@ class TcaTableService76 implements TcaTableServiceInterface $this->tableName = $tableName; $this->tca = &$GLOBALS['TCA'][$this->tableName]; $this->configuration = $configuration; - $this->relationResolver = $relationResolver; } public function getTableName() : string @@ -157,8 +150,6 @@ class TcaTableService76 implements TcaTableServiceInterface public function prepareRecord(array &$record) { - $this->relationResolver->resolveRelationsForRecord($this, $record); - if (isset($record['uid']) && !isset($record['search_identifier'])) { $record['search_identifier'] = $record['uid']; } diff --git a/Documentation/source/changelog.rst b/Documentation/source/changelog.rst index 37b42c6..27e6fab 100644 --- a/Documentation/source/changelog.rst +++ b/Documentation/source/changelog.rst @@ -5,6 +5,7 @@ Changelog :maxdepth: 1 :glob: + changelog/20180424-149-extract-relation-resolver-to-data-processing changelog/20180410-148-keep-sys_language_uid changelog/20180315-134-make-conent-fields-configurable changelog/20180309-25-provide-sys-language-uid diff --git a/Documentation/source/changelog/20180424-149-extract-relation-resolver-to-data-processing.rst b/Documentation/source/changelog/20180424-149-extract-relation-resolver-to-data-processing.rst new file mode 100644 index 0000000..6aff516 --- /dev/null +++ b/Documentation/source/changelog/20180424-149-extract-relation-resolver-to-data-processing.rst @@ -0,0 +1,24 @@ +Breaking Change 149 "Extract RelationResolver to a new DataProcessor" +===================================================================== + +The resolving of relation, based on TCA, is no longer done by the indexer. Instead we +now provide a DataProcessor to solve this job. + +As this makes it necessary to configure the DataProcessor, this is a breaking change. +Before the resolving was done out of the box. + +So why did we change that? The resolving of relations was already implemented before +we added the data processors. As the concept of data processors is far more flexible, +we try to migrate hard coupled components step by step. The benefit of this change is +basically that you can now configure the resolving of relations and far more +important, the order of execution. + +Now it's possible to first copy some fields, e.g. ``starttime`` and ``endtime`` to +further fields and to resolve relations afterwards. As the copied fields are not +configured in TCA, they will be skipped. This way an integrator can keep certain +information as they are. + +Also the processor is now configured as all other processors. You can now optionally +configure fields to not process. + +See :issue:`149` and :issue:`147`. diff --git a/Documentation/source/configuration/dataProcessing/TcaRelationResolvingProcessor.rst b/Documentation/source/configuration/dataProcessing/TcaRelationResolvingProcessor.rst new file mode 100644 index 0000000..a5814d3 --- /dev/null +++ b/Documentation/source/configuration/dataProcessing/TcaRelationResolvingProcessor.rst @@ -0,0 +1,34 @@ +``Codappix\SearchCore\DataProcessing\TcaRelationResolvingProcessor`` +==================================================================== + +Will resolve relations through TCA for record. +The result will be the same as in list view of TYPO3 Backend. E.g. Check boxes will be +resolved to their label, dates will be resolved to human readable representation and +relations will be resolved to their configured labels. + +Combine with CopyToProcessor or exclude certain fields to keep original value for +further processing. + +Mandatory Options: + +``_table`` + The TCA table as found on top level of ``$GLOBALS['TCA']``. + + This will auto filled for indexing through the provided indexers. Still you can + apply processors on results, where no information about the table exists anymore. + +Possible Options: + +``excludeFields`` + Comma separated list of fields to not resolve relations for. + +Example:: + + plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing { + 1 = Codappix\SearchCore\DataProcessing\TcaRelationResolvingProcessor + 1 { + _table = tt_content + excludeFields = starttime, endtime + } + } + diff --git a/Documentation/source/dataprocessors.rst b/Documentation/source/dataprocessors.rst index 67ff655..2023ff2 100644 --- a/Documentation/source/dataprocessors.rst +++ b/Documentation/source/dataprocessors.rst @@ -66,6 +66,7 @@ Available DataProcessors /configuration/dataProcessing/CopyToProcessor /configuration/dataProcessing/GeoPointProcessor /configuration/dataProcessing/RemoveProcessor + /configuration/dataProcessing/TcaRelationResolvingProcessor .. _dataprocessors_plannedDataProcessors: @@ -81,18 +82,3 @@ Planned DataProcessors ``Codappix\SearchCore\DataProcessing\ChannelProcessor`` Will add a configurable channel to the record, e.g. if you have different areas in your website like "products" and "infos". - - ``Codappix\SearchCore\DataProcessing\RelationResolverProcessor`` - Resolves all relations using the TCA. - This is currently done through indexer. - -.. Of course you are able to provide further processors. Just implement -.. ``Codappix\SearchCore\DataProcessing\ProcessorInterface`` and use the FQCN (=Fully qualified -.. class name) as done in the examples above. - -.. By implementing also the same interface as necessary for TYPO3 -.. :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`, you are able to reuse the same code -.. also for Fluid to prepare the same record fetched from DB for your fluid. - -.. Dependency injection is possible inside of processors, as we instantiate through extbase -.. ``ObjectManager``. diff --git a/Tests/Functional/Indexing/TcaIndexer/RelationResolverTest.php b/Tests/Functional/DataProcessing/TcaRelationResolvingProcessorTest.php similarity index 77% rename from Tests/Functional/Indexing/TcaIndexer/RelationResolverTest.php rename to Tests/Functional/DataProcessing/TcaRelationResolvingProcessorTest.php index a491677..c5011df 100644 --- a/Tests/Functional/Indexing/TcaIndexer/RelationResolverTest.php +++ b/Tests/Functional/DataProcessing/TcaRelationResolvingProcessorTest.php @@ -1,5 +1,5 @@ @@ -20,13 +20,13 @@ namespace Codappix\SearchCore\Tests\Indexing\TcaIndexer; * 02110-1301, USA. */ -use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService; +use Codappix\SearchCore\DataProcessing\TcaRelationResolvingProcessor; use Codappix\SearchCore\Tests\Functional\AbstractFunctionalTestCase; use TYPO3\CMS\Backend\Utility\BackendUtility; use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Extbase\Object\ObjectManager; -class RelationResolverTest extends AbstractFunctionalTestCase +class TcaRelationResolvingProcessorTest extends AbstractFunctionalTestCase { /** * @test @@ -37,9 +37,8 @@ class RelationResolverTest extends AbstractFunctionalTestCase $objectManager = GeneralUtility::makeInstance(ObjectManager::class); $table = 'sys_file'; - $subject = $objectManager->get(TcaTableService::class, $table); - $record = BackendUtility::getRecord($table, 1); - $subject->prepareRecord($record); + $subject = $objectManager->get(TcaRelationResolvingProcessor::class); + $record = $subject->processData(BackendUtility::getRecord($table, 1), ['_table' => $table]); $this->assertEquals( [ @@ -60,9 +59,8 @@ class RelationResolverTest extends AbstractFunctionalTestCase $objectManager = GeneralUtility::makeInstance(ObjectManager::class); $table = 'tt_content'; - $subject = $objectManager->get(TcaTableService::class, $table); - $record = BackendUtility::getRecord($table, 1); - $subject->prepareRecord($record); + $subject = $objectManager->get(TcaRelationResolvingProcessor::class); + $record = $subject->processData(BackendUtility::getRecord($table, 1), ['_table' => $table]); $this->assertEquals( 'Insert Plugin', @@ -80,9 +78,8 @@ class RelationResolverTest extends AbstractFunctionalTestCase $objectManager = GeneralUtility::makeInstance(ObjectManager::class); $table = 'tt_content'; - $subject = $objectManager->get(TcaTableService::class, $table); - $record = BackendUtility::getRecord($table, 1); - $subject->prepareRecord($record); + $subject = $objectManager->get(TcaRelationResolvingProcessor::class); + $record = $subject->processData(BackendUtility::getRecord($table, 1), ['_table' => $table]); $this->assertEquals( [ @@ -103,9 +100,8 @@ class RelationResolverTest extends AbstractFunctionalTestCase $objectManager = GeneralUtility::makeInstance(ObjectManager::class); $table = 'tt_content'; - $subject = $objectManager->get(TcaTableService::class, $table); - $record = BackendUtility::getRecord($table, 1); - $subject->prepareRecord($record); + $subject = $objectManager->get(TcaRelationResolvingProcessor::class); + $record = $subject->processData(BackendUtility::getRecord($table, 1), ['_table' => $table]); $this->assertEquals( [ diff --git a/Tests/Functional/Fixtures/BasicSetup.ts b/Tests/Functional/Fixtures/BasicSetup.ts index 840a7b8..a4a26fb 100644 --- a/Tests/Functional/Fixtures/BasicSetup.ts +++ b/Tests/Functional/Fixtures/BasicSetup.ts @@ -22,6 +22,10 @@ plugin { type = keyword } } + + dataProcessing { + 1 = Codappix\SearchCore\DataProcessing\TcaRelationResolvingProcessor + } } pages { @@ -34,6 +38,10 @@ plugin { type = keyword } } + + dataProcessing { + 1 = Codappix\SearchCore\DataProcessing\TcaRelationResolvingProcessor + } } } diff --git a/Tests/Functional/Indexing/TcaIndexerTest.php b/Tests/Functional/Indexing/TcaIndexerTest.php index 85f36a6..472cc15 100644 --- a/Tests/Functional/Indexing/TcaIndexerTest.php +++ b/Tests/Functional/Indexing/TcaIndexerTest.php @@ -77,36 +77,4 @@ class TcaIndexerTest extends AbstractFunctionalTestCase $objectManager->get(TcaIndexer::class, $tableService, $connection)->indexAllDocuments(); } - - /** - * @test - */ - public function sysLanguageIsKept() - { - $this->importDataSet('Tests/Functional/Fixtures/Indexing/TcaIndexer/KeepSysLanguageUid.xml'); - $objectManager = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class); - $tableName = 'tt_content'; - $tableService = $objectManager->get( - TcaTableServiceInterface::class, - $tableName, - $objectManager->get(RelationResolver::class), - $objectManager->get(ConfigurationContainerInterface::class) - ); - - $connection = $this->getMockBuilder(Elasticsearch::class) - ->setMethods(['addDocuments']) - ->disableOriginalConstructor() - ->getMock(); - - $connection->expects($this->once()) - ->method('addDocuments') - ->with( - $this->stringContains('tt_content'), - $this->callback(function ($documents) { - return isset($documents[0]['sys_language_uid']) && $documents[0]['sys_language_uid'] === 2; - }) - ); - - $objectManager->get(TcaIndexer::class, $tableService, $connection)->indexAllDocuments(); - } } diff --git a/Tests/Unit/Bootstrap.php b/Tests/Unit/Bootstrap.php index 0f7b382..3ed0a1c 100644 --- a/Tests/Unit/Bootstrap.php +++ b/Tests/Unit/Bootstrap.php @@ -6,4 +6,6 @@ if (getenv('TYPO3_VERSION') === '~7.6') { $filePath = '.Build/vendor/typo3/cms/typo3/sysext/core/Build/UnitTestsBootstrap.php'; } +date_default_timezone_set('UTC'); + require_once dirname(dirname(__DIR__)) . '/' . $filePath; diff --git a/Tests/Unit/Domain/Index/TcaIndexer/RelationResolverTest.php b/Tests/Unit/DataProcessing/TcaRelationResolvingProcessorTest.php similarity index 53% rename from Tests/Unit/Domain/Index/TcaIndexer/RelationResolverTest.php rename to Tests/Unit/DataProcessing/TcaRelationResolvingProcessorTest.php index 663f7ca..e7abb07 100644 --- a/Tests/Unit/Domain/Index/TcaIndexer/RelationResolverTest.php +++ b/Tests/Unit/DataProcessing/TcaRelationResolvingProcessorTest.php @@ -1,5 +1,5 @@ @@ -20,21 +20,46 @@ namespace Codappix\SearchCore\Tests\Unit\Domain\Index\TcaIndexer; * 02110-1301, USA. */ -use Codappix\SearchCore\Domain\Index\TcaIndexer\RelationResolver; -use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface; +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\DataProcessing\TcaRelationResolvingProcessor; use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; +use TYPO3\CMS\Extbase\Object\ObjectManager; +use \TYPO3\CMS\Core\Utility\GeneralUtility; -class RelationResolverTest extends AbstractUnitTestCase +class TcaRelationResolvingProcessorTest extends AbstractUnitTestCase { /** - * @var RelationResolver + * @var TcaRelationResolvingProcessor */ protected $subject; + /** + * @var ConfigurationContainerInterface + */ + protected $configurationMock; + public function setUp() { parent::setUp(); - $this->subject = new RelationResolver(); + $this->configurationMock = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock(); + + GeneralUtility::makeInstance(\TYPO3\CMS\Extbase\Object\Container\Container::class) + ->registerImplementation( + ConfigurationContainerInterface::class, + get_class($this->configurationMock) + ); + + $this->subject = GeneralUtility::makeInstance(ObjectManager::class) + ->get(TcaRelationResolvingProcessor::class); + } + + /** + * @test + */ + public function exceptionIsThrownIfTableIsNotConfigured() + { + $this->expectException(\InvalidArgumentException::class); + $this->subject->processData([], []); } /** @@ -42,11 +67,17 @@ class RelationResolverTest extends AbstractUnitTestCase */ public function sysLanguageUidZeroIsKept() { - $record = [ + $originalRecord = [ 'sys_language_uid' => '0', ]; + $record = [ + 'sys_language_uid' => 0, + ]; $GLOBALS['TCA'] = [ 'tt_content' => [ + 'ctrl' => [ + 'languageField' => 'sys_language_uid', + ], 'columns' => [ 'sys_language_uid' => [ 'config' => [ @@ -68,19 +99,10 @@ class RelationResolverTest extends AbstractUnitTestCase ], ], ]; - $tableServiceMock = $this->getMockBuilder(TcaTableServiceInterface::class)->getMock(); - $tableServiceMock->expects($this->any()) - ->method('getTableName') - ->willReturn('tt_content'); - $tableServiceMock->expects($this->any()) - ->method('getLanguageUidColumn') - ->willReturn('sys_language_uid'); - $tableServiceMock->expects($this->any()) - ->method('getColumnConfig') - ->willReturn($GLOBALS['TCA']['tt_content']['columns']['sys_language_uid']['config']); - - $this->subject->resolveRelationsForRecord($tableServiceMock, $record); - + $configuration = [ + '_table' => 'tt_content', + ]; + $record = $this->subject->processData($originalRecord, $configuration); $this->assertSame( [ 'sys_language_uid' => 0, @@ -96,7 +118,8 @@ class RelationResolverTest extends AbstractUnitTestCase public function renderTypeInputDateTimeIsHandled() { $originalRecord = [ - 'starttime' => 0, + 'endtime' => 99999999999, + 'starttime' => 1523010960, ]; $record = $originalRecord; $GLOBALS['TCA'] = [ @@ -104,12 +127,24 @@ class RelationResolverTest extends AbstractUnitTestCase 'columns' => [ 'starttime' => [ 'config' => [ + 'type' => 'input', 'default' => 0, - 'eval' => 'datetime', + 'eval' => 'datetime,int', 'renderType' => 'inputDateTime', ], - 'type' => 'input', - 'exclude' => 1, + 'exclude' => true, + 'l10n_display' => 'defaultAsReadonly', + 'l10n_mode' => 'exclude', + 'label' => 'LLL:EXT:lang/Resources/Private/Language/locallang_general.xlf:LGL.starttime', + ], + 'endtime' => [ + 'config' => [ + 'type' => 'input', + 'default' => 0, + 'eval' => 'datetime,int', + 'renderType' => 'inputDateTime', + ], + 'exclude' => true, 'l10n_display' => 'defaultAsReadonly', 'l10n_mode' => 'exclude', 'label' => 'LLL:EXT:lang/Resources/Private/Language/locallang_general.xlf:LGL.starttime', @@ -117,20 +152,18 @@ class RelationResolverTest extends AbstractUnitTestCase ], ], ]; - $tableServiceMock = $this->getMockBuilder(TcaTableServiceInterface::class)->getMock(); - $tableServiceMock->expects($this->any()) - ->method('getTableName') - ->willReturn('tt_content'); - $tableServiceMock->expects($this->any()) - ->method('getColumnConfig') - ->willReturn($GLOBALS['TCA']['tt_content']['columns']['starttime']['config']); - - $this->subject->resolveRelationsForRecord($tableServiceMock, $record); - + $configuration = [ + '_table' => 'tt_content', + 'excludeFields' => 'starttime', + ]; + $record = $this->subject->processData($originalRecord, $configuration); $this->assertSame( - $originalRecord, + [ + 'endtime' => '16-11-38 09:46', + 'starttime' => 1523010960, + ], $record, - 'TCA column configured with renderType inputDateTime was not kept as unix timestamp.' + 'Exclude fields were not respected.' ); } } From ac615d830910cd530cab37017e16b35962947aa1 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 26 Apr 2018 07:52:59 +0200 Subject: [PATCH 74/75] TASK: Raise version number --- Documentation/source/conf.py | 6 +++--- ext_emconf.php | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation/source/conf.py b/Documentation/source/conf.py index aa846e0..ae0b901 100644 --- a/Documentation/source/conf.py +++ b/Documentation/source/conf.py @@ -59,9 +59,9 @@ author = u'Daniel Siepmann' # built documents. # # The short X.Y version. -version = u'0.0.1' +version = u'0.0.4' # The full version, including alpha/beta/rc tags. -release = u'0.0.1' +release = u'0.0.4' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -133,7 +133,7 @@ html_theme_options = { # The name for this set of Sphinx documents. # " v documentation" by default. -#html_title = u'TYPO3 Extension search_core v0.0.1' +#html_title = u'TYPO3 Extension search_core v0.0.4' # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None diff --git a/ext_emconf.php b/ext_emconf.php index 7c3a087..b0b5e65 100644 --- a/ext_emconf.php +++ b/ext_emconf.php @@ -18,7 +18,7 @@ $EM_CONF[$_EXTKEY] = [ ], ], 'state' => 'beta', - 'version' => '0.0.1', + 'version' => '0.0.4', 'author' => 'Daniel Siepmann', 'author_email' => 'coding@daniel-siepmann.de', ]; From a2b0cf2c97241b72eb2787e3875bd4b1fa0cc289 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 26 Apr 2018 09:55:25 +0200 Subject: [PATCH 75/75] TASK: Fix version in install instruction --- Documentation/source/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/source/installation.rst b/Documentation/source/installation.rst index 174d9f7..fc9b479 100644 --- a/Documentation/source/installation.rst +++ b/Documentation/source/installation.rst @@ -9,7 +9,7 @@ Composer The extension can be installed through composer:: - composer require "codappix/search_core" "~0.0.3" + composer require "codappix/search_core" "~0.0.4" Note that you have to allow unstable packages: