diff --git a/.travis.yml b/.travis.yml index ed819a6..811b8f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,6 @@ before_install: language: php php: - - 5.6 - 7.0 - 7.1 - 7.2 @@ -26,7 +25,6 @@ env: - typo3DatabaseHost="127.0.0.1" - typo3DatabaseUsername="travis" - typo3DatabasePassword="" - - TYPO3_VERSION="~7.6" matrix: fast_finish: true diff --git a/Classes/Command/IndexCommandController.php b/Classes/Command/IndexCommandController.php index b14d4b3..70816ed 100644 --- a/Classes/Command/IndexCommandController.php +++ b/Classes/Command/IndexCommandController.php @@ -50,7 +50,6 @@ class IndexCommandController extends CommandController */ public function indexCommand($identifier) { - // TODO: Also allow to index everything? try { $this->indexerFactory->getIndexer($identifier)->indexAllDocuments(); $this->outputLine($identifier . ' was indexed.'); @@ -58,4 +57,19 @@ class IndexCommandController extends CommandController $this->outputLine('No indexer found for: ' . $identifier); } } + + /** + * Will delete the given identifier. + * + * @param string $identifier + */ + public function deleteCommand($identifier) + { + try { + $this->indexerFactory->getIndexer($identifier)->delete(); + $this->outputLine($identifier . ' was deleted.'); + } catch (NoMatchingIndexerException $e) { + $this->outputLine('No indexer found for: ' . $identifier); + } + } } diff --git a/Classes/Configuration/ConfigurationUtility.php b/Classes/Configuration/ConfigurationUtility.php new file mode 100644 index 0000000..29cb721 --- /dev/null +++ b/Classes/Configuration/ConfigurationUtility.php @@ -0,0 +1,69 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Connection\SearchRequestInterface; +use TYPO3\CMS\Fluid\View\StandaloneView; + +class ConfigurationUtility +{ + /** + * Will parse all entries, recursive as fluid template, with request variable set to $searchRequest. + */ + public function replaceArrayValuesWithRequestContent(SearchRequestInterface $searchRequest, array $array) : array + { + array_walk_recursive($array, function (&$value, $key, SearchRequestInterface $searchRequest) { + $template = new StandaloneView(); + $template->assign('request', $searchRequest); + $template->setTemplateSource($value); + $value = $template->render(); + + // As elasticsearch does need some doubles to be send as doubles. + if (is_numeric($value)) { + $value = (float) $value; + } + }, $searchRequest); + + return $array; + } + + /** + * Will check all entries, whether they have a condition and filter entries out, where condition is false. + * Also will remove condition in the end. + */ + public function filterByCondition(array $entries) : array + { + $entries = array_filter($entries, function ($entry) { + return !is_array($entry) + || !array_key_exists('condition', $entry) + || (bool) $entry['condition'] === true + ; + }); + + foreach ($entries as $key => $entry) { + if (is_array($entry) && array_key_exists('condition', $entry)) { + unset($entries[$key]['condition']); + } + } + + return $entries; + } +} diff --git a/Classes/Connection/ConnectionInterface.php b/Classes/Connection/ConnectionInterface.php index a130f9e..59cf9f8 100644 --- a/Classes/Connection/ConnectionInterface.php +++ b/Classes/Connection/ConnectionInterface.php @@ -77,4 +77,13 @@ interface ConnectionInterface * @return SearchResultInterface */ public function search(SearchRequestInterface $searchRequest); + + /** + * Will delete the whole index / db. + * + * @param string $documentType + * + * @return void + */ + public function deleteIndex($documentType); } diff --git a/Classes/Connection/Elasticsearch.php b/Classes/Connection/Elasticsearch.php index 4c66b6a..8a3cb2b 100644 --- a/Classes/Connection/Elasticsearch.php +++ b/Classes/Connection/Elasticsearch.php @@ -156,6 +156,18 @@ class Elasticsearch implements Singleton, ConnectionInterface ); } + public function deleteIndex($documentType) + { + $index = $this->connection->getClient()->getIndex('typo3content'); + + if (! $index->exists()) { + $this->logger->notice('Index did not exist, therefore was not deleted.', [$documentType, 'typo3content']); + return; + } + + $index->delete(); + } + /** * Execute given callback with Elastica Type based on provided documentType * diff --git a/Classes/Controller/SearchController.php b/Classes/Controller/SearchController.php index 0fa7f73..e484fcd 100644 --- a/Classes/Controller/SearchController.php +++ b/Classes/Controller/SearchController.php @@ -56,6 +56,12 @@ class SearchController extends ActionController ] )); } + + if ($this->arguments->hasArgument('searchRequest')) { + $this->arguments->getArgument('searchRequest')->getPropertyMappingConfiguration() + ->allowAllProperties() + ; + } } /** diff --git a/Classes/DataProcessing/CopyToProcessor.php b/Classes/DataProcessing/CopyToProcessor.php index e9eb8cf..2df7438 100644 --- a/Classes/DataProcessing/CopyToProcessor.php +++ b/Classes/DataProcessing/CopyToProcessor.php @@ -25,7 +25,7 @@ namespace Codappix\SearchCore\DataProcessing; */ class CopyToProcessor implements ProcessorInterface { - public function processRecord(array $record, array $configuration) + public function processRecord(array $record, array $configuration) : array { $all = []; @@ -42,7 +42,7 @@ class CopyToProcessor implements ProcessorInterface */ protected function addArray(array &$to, array $from) { - foreach ($from as $property => $value) { + foreach ($from as $value) { if (is_array($value)) { $this->addArray($to, $value); continue; diff --git a/Classes/DataProcessing/GeoPointProcessor.php b/Classes/DataProcessing/GeoPointProcessor.php new file mode 100644 index 0000000..f9256b3 --- /dev/null +++ b/Classes/DataProcessing/GeoPointProcessor.php @@ -0,0 +1,59 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * Adds a new fields, ready to use as GeoPoint field for Elasticsearch. + */ +class GeoPointProcessor implements ProcessorInterface +{ + public function processRecord(array $record, array $configuration) : array + { + if (! $this->canApply($record, $configuration)) { + return $record; + } + + $record[$configuration['to']] = [ + 'lat' => (float) $record[$configuration['lat']], + 'lon' => (float) $record[$configuration['lon']], + ]; + + return $record; + } + + protected function canApply(array $record, array $configuration) : bool + { + if (!isset($record[$configuration['lat']]) + || !is_numeric($record[$configuration['lat']]) + || trim($record[$configuration['lat']]) === '' + ) { + return false; + } + if (!isset($record[$configuration['lon']]) + || !is_numeric($record[$configuration['lon']]) + || trim($record[$configuration['lon']]) === '' + ) { + return false; + } + + return true; + } +} diff --git a/Classes/DataProcessing/ProcessorInterface.php b/Classes/DataProcessing/ProcessorInterface.php index 841edb9..ef051f4 100644 --- a/Classes/DataProcessing/ProcessorInterface.php +++ b/Classes/DataProcessing/ProcessorInterface.php @@ -21,19 +21,14 @@ namespace Codappix\SearchCore\DataProcessing; */ /** - * All DataProcessing Processor should implement this interface, otherwise they - * will not be executed. + * All DataProcessing Processors should implement this interface, otherwise + * they will not be executed. */ interface ProcessorInterface { /** * Processes the given record. * Also retrieves the configuration for this processor instance. - * - * @param array $record - * @param array $configuration - * - * @return array */ - public function processRecord(array $record, array $configuration); + public function processRecord(array $record, array $configuration) : array; } diff --git a/Classes/DataProcessing/RemoveProcessor.php b/Classes/DataProcessing/RemoveProcessor.php new file mode 100644 index 0000000..6e74237 --- /dev/null +++ b/Classes/DataProcessing/RemoveProcessor.php @@ -0,0 +1,44 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use TYPO3\CMS\Core\Utility\GeneralUtility; + +/** + * Removes fields from record. + */ +class RemoveProcessor implements ProcessorInterface +{ + public function processRecord(array $record, array $configuration) : array + { + if (!isset($configuration['fields'])) { + return $record; + } + + foreach (GeneralUtility::trimExplode(',', $configuration['fields'], true) as $field) { + if (array_key_exists($field, $record)) { + unset($record[$field]); + } + } + + return $record; + } +} diff --git a/Classes/Database/Doctrine/Join.php b/Classes/Database/Doctrine/Join.php new file mode 100644 index 0000000..df1a8c6 --- /dev/null +++ b/Classes/Database/Doctrine/Join.php @@ -0,0 +1,50 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +class Join +{ + /** + * @var string + */ + protected $table = ''; + + /** + * @var string + */ + protected $condition = ''; + + public function __construct(string $table, string $condition) + { + $this->table = $table; + $this->condition = $condition; + } + + public function getTable() : string + { + return $this->table; + } + + public function getCondition() : string + { + return $this->condition; + } +} diff --git a/Classes/Database/Doctrine/Where.php b/Classes/Database/Doctrine/Where.php new file mode 100644 index 0000000..6586b8a --- /dev/null +++ b/Classes/Database/Doctrine/Where.php @@ -0,0 +1,50 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +class Where +{ + /** + * @var string + */ + protected $statement = ''; + + /** + * @var array + */ + protected $parameters = []; + + public function __construct(string $statement, array $parameters) + { + $this->statement = $statement; + $this->parameters = $parameters; + } + + public function getStatement() : string + { + return $this->statement; + } + + public function getParameters() : array + { + return $this->parameters; + } +} diff --git a/Classes/Domain/Index/AbstractIndexer.php b/Classes/Domain/Index/AbstractIndexer.php index 143a219..4ccb80e 100644 --- a/Classes/Domain/Index/AbstractIndexer.php +++ b/Classes/Domain/Index/AbstractIndexer.php @@ -23,7 +23,8 @@ namespace Codappix\SearchCore\Domain\Index; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Connection\ConnectionInterface; -use \TYPO3\CMS\Core\Utility\GeneralUtility; +use Codappix\SearchCore\DataProcessing\ProcessorInterface; +use TYPO3\CMS\Core\Utility\GeneralUtility; abstract class AbstractIndexer implements IndexerInterface { @@ -47,6 +48,11 @@ abstract class AbstractIndexer implements IndexerInterface */ protected $logger; + /** + * @var ObjectManagerInterface + */ + protected $objectManager; + /** * Inject log manager to get concrete logger from it. * @@ -62,10 +68,6 @@ abstract class AbstractIndexer implements IndexerInterface $this->identifier = $identifier; } - /** - * @param ConnectionInterface $connection - * @param ConfigurationContainerInterface $configuration - */ public function __construct(ConnectionInterface $connection, ConfigurationContainerInterface $configuration) { $this->connection = $connection; @@ -99,11 +101,19 @@ abstract class AbstractIndexer implements IndexerInterface $this->connection->addDocument($this->getDocumentName(), $record); } catch (NoRecordFoundException $e) { - $this->logger->info('Could not index document.', [$e->getMessage()]); + $this->logger->info('Could not index document. Try to delete it therefore.', [$e->getMessage()]); + $this->connection->deleteDocument($this->getDocumentName(), $identifier); } $this->logger->info('Finish indexing'); } + public function delete() + { + $this->logger->info('Start deletion of index.'); + $this->connection->deleteIndex($this->getDocumentName()); + $this->logger->info('Finish deletion.'); + } + /** * @return \Generator */ @@ -122,6 +132,33 @@ abstract class AbstractIndexer implements IndexerInterface * @param array &$record */ protected function prepareRecord(array &$record) + { + try { + foreach ($this->configuration->get('indexing.' . $this->identifier . '.dataProcessing') as $configuration) { + $className = ''; + if (is_string($configuration)) { + $className = $configuration; + $configuration = []; + } else { + $className = $configuration['_typoScriptNodeValue']; + } + + $dataProcessor = GeneralUtility::makeInstance($className); + if ($dataProcessor instanceof ProcessorInterface) { + $record = $dataProcessor->processRecord($record, $configuration); + } + } + } catch (InvalidArgumentException $e) { + // Nothing to do. + } + + $this->handleAbstract($record); + } + + /** + * @param array &$record + */ + protected function handleAbstract(array &$record) { $record['search_abstract'] = ''; diff --git a/Classes/Domain/Index/IndexerFactory.php b/Classes/Domain/Index/IndexerFactory.php index e228780..2dac06f 100644 --- a/Classes/Domain/Index/IndexerFactory.php +++ b/Classes/Domain/Index/IndexerFactory.php @@ -54,12 +54,9 @@ class IndexerFactory implements Singleton } /** - * @param string $identifier - * - * @return IndexerInterface * @throws NoMatchingIndexer */ - public function getIndexer($identifier) + public function getIndexer(string $identifier) : IndexerInterface { try { return $this->buildIndexer($this->configuration->get('indexing.' . $identifier . '.indexer'), $identifier); @@ -73,13 +70,9 @@ class IndexerFactory implements Singleton } /** - * @param string $indexerClass - * @param string $identifier - * - * @return IndexerInterface * @throws NoMatchingIndexer */ - protected function buildIndexer($indexerClass, $identifier) + protected function buildIndexer(string $indexerClass, string $identifier) : IndexerInterface { $indexer = null; if (is_subclass_of($indexerClass, TcaIndexer\PagesIndexer::class) diff --git a/Classes/Domain/Index/IndexerInterface.php b/Classes/Domain/Index/IndexerInterface.php index 5a4ca6c..72ebb9d 100644 --- a/Classes/Domain/Index/IndexerInterface.php +++ b/Classes/Domain/Index/IndexerInterface.php @@ -33,9 +33,9 @@ interface IndexerInterface public function indexAllDocuments(); /** - * Fetches a single document from the indexerService and pushes it to the connection. + * Fetches a single document and pushes it to the connection. * - * @param string $identifier identifier, the indexer needs to identify a single document + * @param string $identifier * * @return void */ @@ -49,4 +49,11 @@ interface IndexerInterface * @return void */ public function setIdentifier($identifier); + + /** + * Delete the whole index. + * + * @return void + */ + public function delete(); } diff --git a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php index b5ddf56..1108676 100644 --- a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php +++ b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php @@ -58,9 +58,6 @@ class PagesIndexer extends TcaIndexer $this->configuration = $configuration; } - /** - * @param array &$record - */ protected function prepareRecord(array &$record) { $possibleTitleFields = ['nav_title', 'tx_tqseo_pagetitle_rel', 'title']; @@ -80,11 +77,7 @@ class PagesIndexer extends TcaIndexer parent::prepareRecord($record); } - /** - * @param int $uid - * @return [] - */ - protected function fetchContentForPage($uid) + protected function fetchContentForPage(int $uid) : array { $contentElements = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows( $this->contentTableService->getFields(), @@ -118,31 +111,17 @@ class PagesIndexer extends TcaIndexer ]; } - /** - * @param int $uidOfContentElement - * @return array - */ - protected function getContentElementImages($uidOfContentElement) + protected function getContentElementImages(int $uidOfContentElement) : array { return $this->fetchSysFileReferenceUids($uidOfContentElement, 'tt_content', 'image'); } - /** - * @param int $uid - * @return [] - */ - protected function fetchMediaForPage($uid) + protected function fetchMediaForPage(int $uid) : array { return $this->fetchSysFileReferenceUids($uid, 'pages', 'media'); } - /** - * @param int $uid - * @param string $tablename - * @param string $fieldname - * @return [] - */ - protected function fetchSysFileReferenceUids($uid, $tablename, $fieldname) + protected function fetchSysFileReferenceUids(int $uid, string $tablename, string $fieldname) : array { $imageRelationUids = []; $imageRelations = $this->fileRepository->findByRelation($tablename, $fieldname, $uid); diff --git a/Classes/Domain/Index/TcaIndexer/RelationResolver.php b/Classes/Domain/Index/TcaIndexer/RelationResolver.php index b6fc287..6af330e 100644 --- a/Classes/Domain/Index/TcaIndexer/RelationResolver.php +++ b/Classes/Domain/Index/TcaIndexer/RelationResolver.php @@ -20,8 +20,10 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer; * 02110-1301, USA. */ +use Codappix\SearchCore\Utility\FrontendUtility; use TYPO3\CMS\Backend\Utility\BackendUtility; use TYPO3\CMS\Core\SingletonInterface as Singleton; +use TYPO3\CMS\Core\Utility\GeneralUtility; /** * Resolves relations from TCA using TCA. @@ -38,13 +40,15 @@ class RelationResolver implements Singleton if ($column === 'pid') { continue; } - $record[$column] = BackendUtility::getProcessedValueExtra( - $service->getTableName(), - $column, - $record[$column], - 0, - $record['uid'] - ); + + $record[$column] = GeneralUtility::makeInstance($this->getUtilityForMode()) + ::getProcessedValueExtra( + $service->getTableName(), + $column, + $record[$column], + 0, + $record['uid'] + ); try { $config = $service->getColumnConfig($column); @@ -75,7 +79,7 @@ class RelationResolver implements Singleton return []; } - protected function isRelation(array &$config) + protected function isRelation(array &$config) : bool { return isset($config['foreign_table']) || (isset($config['renderType']) && $config['renderType'] !== 'selectSingle') @@ -83,13 +87,22 @@ class RelationResolver implements Singleton ; } - protected function resolveForeignDbValue($value) + protected function resolveForeignDbValue(string $value) : array { return array_map('trim', explode(';', $value)); } - protected function resolveInlineValue($value) + protected function resolveInlineValue(string $value) : array { return array_map('trim', explode(',', $value)); } + + protected function getUtilityForMode() : string + { + if (TYPO3_MODE === 'BE') { + return BackendUtility::class; + } + + return FrontendUtility::class; + } } diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index e68a94f..4b1f5b9 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -22,7 +22,6 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\InvalidArgumentException as InvalidConfigurationArgumentException; -use Codappix\SearchCore\DataProcessing\ProcessorInterface; use Codappix\SearchCore\Domain\Index\IndexingException; use TYPO3\CMS\Backend\Utility\BackendUtility; use TYPO3\CMS\Core\Utility\GeneralUtility; @@ -109,7 +108,7 @@ class TcaTableService /** * @return string */ - public function getTableName() + public function getTableName() : string { return $this->tableName; } @@ -117,7 +116,7 @@ class TcaTableService /** * @return string */ - public function getTableClause() + public function getTableClause() : string { if ($this->tableName === 'pages') { return $this->tableName; @@ -128,9 +127,6 @@ class TcaTableService /** * Filter the given records by root line blacklist settings. - * - * @param array &$records - * @return void */ public function filterRecordsByRootLineBlacklist(array &$records) { @@ -144,23 +140,11 @@ class TcaTableService /** * Adjust record accordingly to configuration. - * @param array &$record */ public function prepareRecord(array &$record) { $this->relationResolver->resolveRelationsForRecord($this, $record); - try { - foreach ($this->configuration->get('indexing.' . $this->tableName . '.dataProcessing') as $configuration) { - $dataProcessor = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance($configuration['_typoScriptNodeValue']); - if ($dataProcessor instanceof ProcessorInterface) { - $record = $dataProcessor->processRecord($record, $configuration); - } - } - } catch (InvalidConfigurationArgumentException $e) { - // Nothing to do. - } - if (isset($record['uid']) && !isset($record['search_identifier'])) { $record['search_identifier'] = $record['uid']; } @@ -169,10 +153,7 @@ class TcaTableService } } - /** - * @return string - */ - public function getWhereClause() + public function getWhereClause() : string { $whereClause = '1=1' . BackendUtility::BEenableFields($this->tableName) @@ -204,17 +185,17 @@ class TcaTableService return $whereClause; } - /** - * @return string - */ - public function getFields() + public function getFields() : string { $fields = array_merge( ['uid','pid'], array_filter( array_keys($this->tca['columns']), function ($columnName) { - return !$this->isSystemField($columnName); + return !$this->isSystemField($columnName) + && !$this->isUserField($columnName) + && !$this->isPassthroughField($columnName) + ; } ) ); @@ -228,10 +209,27 @@ class TcaTableService } /** - * @param string - * @return bool + * Generate SQL for TYPO3 as a system, to make sure only available records + * are fetched. */ - protected function isSystemField($columnName) + public function getSystemWhereClause() : string + { + $whereClause = '1=1' + . BackendUtility::BEenableFields($this->tableName) + . BackendUtility::deleteClause($this->tableName) + . ' AND pages.no_search = 0' + ; + + if ($this->tableName !== 'pages') { + $whereClause .= BackendUtility::BEenableFields('pages') + . BackendUtility::deleteClause('pages') + ; + } + + return $whereClause; + } + + protected function isSystemField(string $columnName) : bool { $systemFields = [ // Versioning fields, @@ -250,12 +248,22 @@ class TcaTableService return in_array($columnName, $systemFields); } + protected function isUserField(string $columnName) : bool + { + $config = $this->getColumnConfig($columnName); + return isset($config['type']) && $config['type'] === 'user'; + } + + protected function isPassthroughField(string $columnName) : bool + { + $config = $this->getColumnConfig($columnName); + return isset($config['type']) && $config['type'] === 'passthrough'; + } + /** - * @param string $columnName - * @return array * @throws InvalidArgumentException */ - public function getColumnConfig($columnName) + public function getColumnConfig(string $columnName) : array { if (!isset($this->tca['columns'][$columnName])) { throw new InvalidArgumentException( @@ -274,11 +282,8 @@ class TcaTableService * Also further TYPO3 mechanics are taken into account. Does a valid root * line exist, is page inside a recycler, is inherited start- endtime * excluded, etc. - * - * @param array &$record - * @return bool */ - protected function isRecordBlacklistedByRootline(array &$record) + protected function isRecordBlacklistedByRootline(array &$record) : bool { $pageUid = $record['pid']; if ($this->tableName === 'pages') { @@ -332,20 +337,16 @@ class TcaTableService /** * Checks whether any page uids are black listed. - * - * @return bool */ - protected function isBlackListedRootLineConfigured() + protected function isBlackListedRootLineConfigured() : bool { return (bool) $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist'); } /** * Get the list of black listed root line page uids. - * - * @return array */ - protected function getBlackListedRootLine() + protected function getBlackListedRootLine() : array { return GeneralUtility::intExplode(',', $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist')); } diff --git a/Classes/Domain/Model/SearchRequest.php b/Classes/Domain/Model/SearchRequest.php index 39e477c..d751ce8 100644 --- a/Classes/Domain/Model/SearchRequest.php +++ b/Classes/Domain/Model/SearchRequest.php @@ -92,7 +92,8 @@ class SearchRequest implements SearchRequestInterface */ public function setFilter(array $filter) { - $this->filter = array_filter(array_map('strval', $filter)); + $filter = \TYPO3\CMS\Core\Utility\ArrayUtility::removeArrayEntryByValue($filter, ''); + $this->filter = \TYPO3\CMS\Extbase\Utility\ArrayUtility::removeEmptyElementsRecursively($filter); } /** diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index c4b66c9..e193e64 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -21,6 +21,7 @@ namespace Codappix\SearchCore\Domain\Search; */ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Configuration\ConfigurationUtility; use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Connection\Elasticsearch\Query; use Codappix\SearchCore\Connection\SearchRequestInterface; @@ -40,37 +41,31 @@ class QueryFactory protected $configuration; /** - * @param \TYPO3\CMS\Core\Log\LogManager $logManager - * @param ConfigurationContainerInterface $configuration + * @var ConfigurationUtility */ + protected $configurationUtility; + public function __construct( \TYPO3\CMS\Core\Log\LogManager $logManager, - ConfigurationContainerInterface $configuration + ConfigurationContainerInterface $configuration, + ConfigurationUtility $configurationUtility ) { $this->logger = $logManager->getLogger(__CLASS__); $this->configuration = $configuration; + $this->configurationUtility = $configurationUtility; } /** * TODO: This is not in scope Elasticsearch, therefore it should not return * \Elastica\Query, but decide to use a more specific QueryFactory like * ElasticaQueryFactory, once the second query is added? - * - * @param SearchRequestInterface $searchRequest - * - * @return \Elastica\Query */ - public function create(SearchRequestInterface $searchRequest) + public function create(SearchRequestInterface $searchRequest) : \Elastica\Query { return $this->createElasticaQuery($searchRequest); } - /** - * @param SearchRequestInterface $searchRequest - * - * @return \Elastica\Query - */ - protected function createElasticaQuery(SearchRequestInterface $searchRequest) + protected function createElasticaQuery(SearchRequestInterface $searchRequest) : \Elastica\Query { $query = []; $this->addSize($searchRequest, $query); @@ -78,6 +73,8 @@ class QueryFactory $this->addBoosts($searchRequest, $query); $this->addFilter($searchRequest, $query); $this->addFacets($searchRequest, $query); + $this->addFields($searchRequest, $query); + $this->addSort($searchRequest, $query); // Use last, as it might change structure of query. // Better approach would be something like DQL to generate query and build result in the end. @@ -87,10 +84,6 @@ class QueryFactory return new \Elastica\Query($query); } - /** - * @param SearchRequestInterface $searchRequest - * @param array $query - */ protected function addSize(SearchRequestInterface $searchRequest, array &$query) { $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ @@ -99,10 +92,6 @@ class QueryFactory ]); } - /** - * @param SearchRequestInterface $searchRequest - * @param array $query - */ protected function addSearch(SearchRequestInterface $searchRequest, array &$query) { if (trim($searchRequest->getSearchTerm()) === '') { @@ -112,7 +101,7 @@ class QueryFactory $matchExpression = [ 'type' => 'most_fields', 'query' => $searchRequest->getSearchTerm(), - 'fields' => GeneralUtility::trimExplode(',', $this->configuration->get('searching.fields')), + 'fields' => GeneralUtility::trimExplode(',', $this->configuration->get('searching.fields.query')), ]; $minimumShouldMatch = $this->configuration->getIfExists('searching.minimumShouldMatch'); @@ -123,10 +112,6 @@ class QueryFactory $query = ArrayUtility::setValueByPath($query, 'query.bool.must.0.multi_match', $matchExpression); } - /** - * @param SearchRequestInterface $searchRequest - * @param array $query - */ protected function addBoosts(SearchRequestInterface $searchRequest, array &$query) { try { @@ -159,9 +144,6 @@ class QueryFactory } } - /** - * @param array $query - */ protected function addFactorBoost(array &$query) { try { @@ -176,38 +158,83 @@ class QueryFactory } } - /** - * @param SearchRequestInterface $searchRequest - * @param array $query - */ + protected function addFields(SearchRequestInterface $searchRequest, array &$query) + { + try { + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ + 'stored_fields' => GeneralUtility::trimExplode(',', $this->configuration->get('searching.fields.stored_fields'), true), + ]); + } catch (InvalidArgumentException $e) { + // Nothing configured + } + + try { + $scriptFields = $this->configuration->get('searching.fields.script_fields'); + $scriptFields = $this->configurationUtility->replaceArrayValuesWithRequestContent($searchRequest, $scriptFields); + $scriptFields = $this->configurationUtility->filterByCondition($scriptFields); + if ($scriptFields !== []) { + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['script_fields' => $scriptFields]); + } + } catch (InvalidArgumentException $e) { + // Nothing configured + } + } + + protected function addSort(SearchRequestInterface $searchRequest, array &$query) + { + $sorting = $this->configuration->getIfExists('searching.sort') ?: []; + $sorting = $this->configurationUtility->replaceArrayValuesWithRequestContent($searchRequest, $sorting); + $sorting = $this->configurationUtility->filterByCondition($sorting); + if ($sorting !== []) { + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['sort' => $sorting]); + } + } + protected function addFilter(SearchRequestInterface $searchRequest, array &$query) { if (! $searchRequest->hasFilter()) { return; } - $terms = []; + $filter = []; foreach ($searchRequest->getFilter() as $name => $value) { - $terms[] = [ + $filter[] = $this->buildFilter( + $name, + $value, + $this->configuration->getIfExists('searching.mapping.filter.' . $name) ?: [] + ); + } + + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ + 'query' => [ + 'bool' => [ + 'filter' => $filter, + ], + ], + ]); + } + + protected function buildFilter(string $name, $value, array $config) : array + { + if ($config === []) { + return [ 'term' => [ $name => $value, ], ]; } - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ - 'query' => [ - 'bool' => [ - 'filter' => $terms, - ], - ], - ]); + $filter = []; + + if (isset($config['fields'])) { + foreach ($config['fields'] as $elasticField => $inputField) { + $filter[$elasticField] = $value[$inputField]; + } + } + + return [$config['field'] => $filter]; } - /** - * @param SearchRequestInterface $searchRequest - * @param array $query - */ protected function addFacets(SearchRequestInterface $searchRequest, array &$query) { foreach ($searchRequest->getFacets() as $facet) { diff --git a/Classes/Domain/Search/SearchService.php b/Classes/Domain/Search/SearchService.php index 384f6aa..120ab3b 100644 --- a/Classes/Domain/Search/SearchService.php +++ b/Classes/Domain/Search/SearchService.php @@ -26,6 +26,7 @@ use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Connection\SearchRequestInterface; use Codappix\SearchCore\Connection\SearchResultInterface; use Codappix\SearchCore\Domain\Model\FacetRequest; +use TYPO3\CMS\Core\Utility\ArrayUtility; use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; /** @@ -123,10 +124,16 @@ class SearchService protected function addConfiguredFilters(SearchRequestInterface $searchRequest) { try { - $searchRequest->setFilter(array_merge( - $searchRequest->getFilter(), - $this->configuration->get('searching.filter') - )); + $filter = $searchRequest->getFilter(); + + ArrayUtility::mergeRecursiveWithOverrule( + $filter, + $this->configuration->get('searching.filter'), + true, + false + ); + + $searchRequest->setFilter($filter); } catch (InvalidArgumentException $e) { // Nothing todo, no filter configured. } diff --git a/Classes/Utility/FrontendUtility.php b/Classes/Utility/FrontendUtility.php new file mode 100644 index 0000000..1282421 --- /dev/null +++ b/Classes/Utility/FrontendUtility.php @@ -0,0 +1,39 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use TYPO3\CMS\Backend\Utility\BackendUtility; +use TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController; + +/** + * Overwrite BackendUtility to use in frontend. + * LanguageService was only usable in backend. + */ +class FrontendUtility extends BackendUtility +{ + /** + * @return TypoScriptFrontendController + */ + protected static function getLanguageService() + { + return $GLOBALS['TSFE']; + } +} diff --git a/Configuration/TypoScript/setup.txt b/Configuration/TypoScript/setup.txt index 67612e1..1a1577f 100644 --- a/Configuration/TypoScript/setup.txt +++ b/Configuration/TypoScript/setup.txt @@ -21,6 +21,12 @@ plugin { abstractFields = {$plugin.tx_searchcore.settings.indexing.pages.abstractFields} } } + + searching { + fields { + query = _all + } + } } } } diff --git a/Documentation/source/concepts.rst b/Documentation/source/concepts.rst index 0ce3a0e..0fe6c5d 100644 --- a/Documentation/source/concepts.rst +++ b/Documentation/source/concepts.rst @@ -28,3 +28,13 @@ The indexing is done by one of the available indexer. For each identifier it's p the indexer to use. Also it's possible to write custom indexer to use. Currently only the :ref:`TcaIndexer` is provided. + +.. _concepts_indexing_dataprocessing: + +DataProcessing +^^^^^^^^^^^^^^ + +Before data is transfered to search service, it can be processed by "DataProcessors" like already +known by :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing` of :ref:`t3tsref:cobj-fluidtemplate`. + +Configuration is done through TypoScript, see :ref:`dataProcessing`. diff --git a/Documentation/source/conf.py b/Documentation/source/conf.py index a708209..1419689 100644 --- a/Documentation/source/conf.py +++ b/Documentation/source/conf.py @@ -304,6 +304,7 @@ texinfo_documents = [ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = { 't3tcaref': ('https://docs.typo3.org/typo3cms/TCAReference/', None), + 't3tsref': ('https://docs.typo3.org/typo3cms/TyposcriptReference/', None), } extlinks = { 'project': ('https://github.com/Codappix/search_core/projects/%s', 'Github project: '), diff --git a/Documentation/source/configuration.rst b/Documentation/source/configuration.rst index 549ca68..eca5ba4 100644 --- a/Documentation/source/configuration.rst +++ b/Documentation/source/configuration.rst @@ -36,330 +36,14 @@ Here is the example default configuration that's provided through static include Options ------- -The following section contains the different options, e.g. for :ref:`connections` and -:ref:`indexer`: ``plugin.tx_searchcore.settings.connection`` or -``plugin.tx_searchcore.settings.indexing``. +The following sections contains the different options grouped by their applied area, e.g. for +:ref:`connections` and :ref:`indexer`: ``plugin.tx_searchcore.settings.connection`` or +``plugin.tx_searchcore.settings.indexing``: -.. _configuration_options_connection: +.. toctree:: + :maxdepth: 1 + :glob: -connections -^^^^^^^^^^^ - -Holds settings regarding the different possible connections for search services like Elasticsearch -or Solr. - -Configured as:: - - plugin { - tx_searchcore { - settings { - connections { - connectionName { - // the settings - } - } - } - } - } - -Where ``connectionName`` is one of the available :ref:`connections`. - -The following settings are available. For each setting its documented which connection consumes it. - -.. _host: - -``host`` -"""""""" - - Used by: :ref:`Elasticsearch`. - - The host, e.g. ``localhost`` or an IP where the search service is reachable from TYPO3 - installation. - - Example:: - - plugin.tx_searchcore.settings.connections.elasticsearch.host = localhost - -.. _port: - -``port`` -"""""""" - - Used by: :ref:`Elasticsearch`. - - The port where search service is reachable. E.g. default ``9200`` for Elasticsearch. - - Example:: - - plugin.tx_searchcore.settings.connections.elasticsearch.port = 9200 - - -.. _configuration_options_index: - -Indexing -^^^^^^^^ - -Holds settings regarding the indexing, e.g. of TYPO3 records, to search services. - -Configured as:: - - plugin { - tx_searchcore { - settings { - indexing { - identifier { - indexer = FullyQualifiedClassname - // the settings - } - } - } - } - } - -Where ``identifier`` is up to you, but should match table names to make :ref:`TcaIndexer` work. - -The following settings are available. For each setting its documented which indexer consumes it. - -.. _rootLineBlacklist: - -``rootLineBlacklist`` -""""""""""""""""""""" - - Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. - - Defines a blacklist of page uids. Records below any of these pages, or subpages, are not - indexed. This allows you to define areas that should not be indexed. - The page attribute *No Search* is also taken into account to prevent indexing records from only one - page without recursion. - - Contains a comma separated list of page uids. Spaces are trimmed. - - Example:: - - plugin.tx_searchcore.settings.indexing..rootLineBlacklist = 3, 10, 100 - -Also it's possible to define some behaviour for the different document types. In context of TYPO3 -tables are used as document types 1:1. It's possible to configure different tables. The following -options are available: - -.. _additionalWhereClause: - -``additionalWhereClause`` -""""""""""""""""""""""""" - - Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. - - Add additional SQL to where clauses to determine indexable records from the table. This way you - can exclude specific records like ``tt_content`` records with specific ``CType`` values or - something else. E.g. you can add a new field to the table to exclude records from indexing. - - Example:: - - plugin.tx_searchcore.settings.indexing..additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu') - - .. attention:: - - Make sure to prefix all fields with the corresponding table name. The selection from - database will contain joins and can lead to SQL errors if a field exists in multiple tables. - -.. _abstractFields: - -``abstractFields`` -""""""""""""""""""""""""" - - Used by: :ref:`PagesIndexer`. - - Define which field should be used to provide the auto generated field "search_abstract". - The fields have to exist in the record to be indexed. Therefore fields like ``content`` are also - possible. - - Example:: - - # As last fallback we use the content of the page - plugin.tx_searchcore.settings.indexing..abstractFields := addToList(content) - - Default:: - - abstract, description, bodytext - -.. _mapping: - -``mapping`` -""""""""""" - - Used by: Elasticsearch connection while indexing. - - Define mapping for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/mapping.html - You are able to define the mapping for each property / columns. - - Example:: - - plugin.tx_searchcore.settings.indexing.tt_content.mapping { - CType { - type = keyword - } - } - - The above example will define the ``CType`` field of ``tt_content`` as ``type: keyword``. This - makes building a facet possible. - - -.. _index: - -``index`` -""""""""" - - Used by: Elasticsearch connection while indexing. - - Define index for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/indices-create-index.html - - Example:: - - plugin.tx_searchcore.settings.indexing.tt_content.index { - analysis { - analyzer { - ngram4 { - type = custom - tokenizer = ngram4 - char_filter = html_strip - filter = lowercase, asciifolding - } - } - - tokenizer { - ngram4 { - type = ngram - min_gram = 4 - max_gram = 4 - } - } - } - } - - ``char_filter`` and ``filter`` are a comma separated list of options. - -.. _configuration_options_search: - -Searching -^^^^^^^^^ - -.. _size: - -``size`` -"""""""" - - Used by: Elasticsearch connection while building search query. - - Defined how many search results should be fetched to be available in search result. - - Example:: - - plugin.tx_searchcore.settings.searching.size = 50 - - Default if not configured is 10. - -.. _facets: - -``facets`` -""""""""""" - - Used by: Elasticsearch connection while building search query. - - Define aggregations for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-aggregations-bucket-terms-aggregation.html - Currently only the term facet is provided. - - Example:: - - plugin.tx_searchcore.settings.searching.facets { - contentTypes { - field = CType - } - } - - The above example will provide a facet with options for all found ``CType`` results together - with a count. - -.. _filter: - -``filter`` -""""""""""" - - Used by: While building search request. - - Define filter that should be set for all requests. - - Example:: - - plugin.tx_searchcore.settings.searching.filter { - property = value - } - - For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``keyword``. - -.. _minimumShouldMatch: - -``minimumShouldMatch`` -"""""""""""""""""""""" - - Used by: Elasticsearch connection while building search query. - - Define the minimum match for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-minimum-should-match.html - - Example:: - - plugin.tx_searchcore.settings.searching.minimumShouldMatch = 50% - -.. _boost: - -``boost`` -""""""""" - - Used by: Elasticsearch connection while building search query. - - Define fields that should boost the score for results. - - Example:: - - plugin.tx_searchcore.settings.searching.boost { - search_title = 3 - search_abstract = 1.5 - } - - For further information take a look at - https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_boosting_query_clauses.html - -.. _fieldValueFactor: - -``fieldValueFactor`` -"""""""""""""""""""" - - Used by: Elasticsearch connection while building search query. - - Define a field to use as a factor for scoring. The configuration is passed through to elastic - search ``field_value_factor``, see: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor - - Example:: - - plugin.tx_searchcore.settings.searching.field_value_factor { - field = rootlineLevel - modifier = reciprocal - factor = 2 - missing = 1 - } - -.. _mode: - -``mode`` -"""""""" - - Used by: Controller while preparing action. - - Define to switch from search to filter mode. - - Example:: - - plugin.tx_searchcore.settings.searching { - mode = filter - } - - Only ``filter`` is allowed as value. Will submit an empty query to switch to filter mode. + configuration/connections + configuration/indexing + configuration/searching diff --git a/Documentation/source/configuration/connections.rst b/Documentation/source/configuration/connections.rst new file mode 100644 index 0000000..5819730 --- /dev/null +++ b/Documentation/source/configuration/connections.rst @@ -0,0 +1,55 @@ +.. _configuration_options_connection: + +Connections +=========== + +Holds settings regarding the different possible connections for search services like Elasticsearch +or Solr. + +Configured as:: + + plugin { + tx_searchcore { + settings { + connections { + connectionName { + // the settings + } + } + } + } + } + +Where ``connectionName`` is one of the available :ref:`connections`. + +The following settings are available. For each setting its documented which connection consumes it. + +.. _host: + +``host`` +-------- + +Used by: :ref:`Elasticsearch`. + +The host, e.g. ``localhost`` or an IP where the search service is reachable from TYPO3 +installation. + +Example:: + + plugin.tx_searchcore.settings.connections.elasticsearch.host = localhost + +.. _port: + +``port`` +-------- + +Used by: :ref:`Elasticsearch`. + +The port where search service is reachable. E.g. default ``9200`` for Elasticsearch. + +Example:: + + plugin.tx_searchcore.settings.connections.elasticsearch.port = 9200 + + + diff --git a/Documentation/source/configuration/dataProcessing/CopyToProcessor.rst b/Documentation/source/configuration/dataProcessing/CopyToProcessor.rst new file mode 100644 index 0000000..6d83d70 --- /dev/null +++ b/Documentation/source/configuration/dataProcessing/CopyToProcessor.rst @@ -0,0 +1,23 @@ +``Codappix\SearchCore\DataProcessing\CopyToProcessor`` +====================================================== + +Will copy contents of fields to other fields. + +Possible Options: + +``to`` + Defines the field to copy the values into. All values not false will be copied at the moment. + +Example:: + + plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing { + 1 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 1 { + to = all + } + 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 2 { + to = spellcheck + } + } + diff --git a/Documentation/source/configuration/dataProcessing/GeoPointProcessor.rst b/Documentation/source/configuration/dataProcessing/GeoPointProcessor.rst new file mode 100644 index 0000000..a19928f --- /dev/null +++ b/Documentation/source/configuration/dataProcessing/GeoPointProcessor.rst @@ -0,0 +1,27 @@ +``Codappix\SearchCore\DataProcessing\GeoPointProcessor`` +======================================================== + +Will create a new field, ready to use as GeoPoint field for Elasticsearch. + +Possible Options: + +``to`` + Defines the field to create as GeoPoint. +``lat`` + Defines the field containing the latitude. +``lon`` + Defines the field containing the longitude. + +Example:: + + plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing { + 1 = Codappix\SearchCore\DataProcessing\GeoPointProcessor + 1 { + to = location + lat = lat + lon = lng + } + } + +The above example will create a new field ``location`` as GeoPoint with latitude fetched from field +``lat`` and longitude fetched from field ``lng``. diff --git a/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst b/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst new file mode 100644 index 0000000..c8653a5 --- /dev/null +++ b/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst @@ -0,0 +1,23 @@ +``Codappix\SearchCore\DataProcessing\RemoveProcessor`` +====================================================== + +Will remove fields from record, e.g. if you do not want to sent them to elasticsearch at all. + +Possible Options: + +``fields`` + Comma separated list of fields to remove from record. + +Example:: + + plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing { + 1 = Codappix\SearchCore\DataProcessing\RemoveProcessor + 1 { + fields = description + } + 2 = Codappix\SearchCore\DataProcessing\RemoveProcessor + 2 { + fields = description, another_field + } + } + diff --git a/Documentation/source/configuration/indexing.rst b/Documentation/source/configuration/indexing.rst new file mode 100644 index 0000000..b605008 --- /dev/null +++ b/Documentation/source/configuration/indexing.rst @@ -0,0 +1,207 @@ +.. _configuration_options_index: + +Indexing +======== + +Holds settings regarding the indexing, e.g. of TYPO3 records, to search services. + +Configured as:: + + plugin { + tx_searchcore { + settings { + indexing { + identifier { + indexer = FullyQualifiedClassname + // the settings + } + } + } + } + } + +Where ``identifier`` is up to you, but should match table names to make :ref:`TcaIndexer` work. + +The following settings are available. For each setting its documented which indexer consumes it. + +.. _rootLineBlacklist: + +rootLineBlacklist +----------------- + +Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. + +Defines a blacklist of page uids. Records below any of these pages, or subpages, are not +indexed. This allows you to define areas that should not be indexed. +The page attribute *No Search* is also taken into account to prevent indexing records from only one +page without recursion. + +Contains a comma separated list of page uids. Spaces are trimmed. + +Example:: + + plugin.tx_searchcore.settings.indexing..rootLineBlacklist = 3, 10, 100 + +Also it's possible to define some behaviour for the different document types. In context of TYPO3 +tables are used as document types 1:1. It's possible to configure different tables. The following +options are available: + +.. _additionalWhereClause: + +additionalWhereClause +--------------------- + +Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. + +Add additional SQL to where clauses to determine indexable records from the table. This way you +can exclude specific records like ``tt_content`` records with specific ``CType`` values or +something else. E.g. you can add a new field to the table to exclude records from indexing. + +Example:: + + plugin.tx_searchcore.settings.indexing..additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu') + +.. attention:: + + Make sure to prefix all fields with the corresponding table name. The selection from + database will contain joins and can lead to SQL errors if a field exists in multiple tables. + +.. _abstractFields: + +abstractFields +-------------- + +Used by: :ref:`PagesIndexer`. + +Define which field should be used to provide the auto generated field "search_abstract". +The fields have to exist in the record to be indexed. Therefore fields like ``content`` are also +possible. + +Example:: + + # As last fallback we use the content of the page + plugin.tx_searchcore.settings.indexing..abstractFields := addToList(content) + +Default:: + + abstract, description, bodytext + +.. _mapping: + +mapping +------- + +Used by: Elasticsearch connection while indexing. + +Define mapping for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/mapping.html +You are able to define the mapping for each property / columns. + +Example:: + + plugin.tx_searchcore.settings.indexing.tt_content.mapping { + CType { + type = keyword + } + } + +The above example will define the ``CType`` field of ``tt_content`` as ``type: keyword``. This +makes building a facet possible. + +.. _index: + +index +----- + +Used by: Elasticsearch connection while indexing. + +Define index for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/indices-create-index.html + +Example:: + + plugin.tx_searchcore.settings.indexing.tt_content.index { + analysis { + analyzer { + ngram4 { + type = custom + tokenizer = ngram4 + char_filter = html_strip + filter = lowercase, asciifolding + } + } + + tokenizer { + ngram4 { + type = ngram + min_gram = 4 + max_gram = 4 + } + } + } + } + +``char_filter`` and ``filter`` are a comma separated list of options. + +.. _dataProcessing: + +dataProcessing +-------------- + +Used by: All connections while indexing. + +Configure modifications on each document before sending it to the configured connection. Same as +provided by TYPO3 for :ref:`t3tsref:cobj-fluidtemplate` through +:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`. + +All processors are applied in configured order. Allowing to work with already processed data. + +Example:: + + plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing { + 1 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 1 { + to = search_spellcheck + } + + 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 2 { + to = search_all + } + } + +The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards +all fields, including ``search_spellcheck`` will be copied to ``search_all``. +E.g. used to index all information into a field for :ref:`spellchecking` or searching with +different :ref:`mapping`. + +The following Processor are available: + +.. toctree:: + :maxdepth: 1 + :glob: + + dataProcessing/CopyToProcessor + dataProcessing/RemoveProcessor + dataProcessing/GeoPointProcessor + +The following Processor are planned: + + ``Codappix\SearchCore\DataProcessing\ReplaceProcessor`` + Will execute a search and replace on configured fields. + + ``Codappix\SearchCore\DataProcessing\RootLevelProcessor`` + Will attach the root level to the record. + + ``Codappix\SearchCore\DataProcessing\ChannelProcessor`` + Will add a configurable channel to the record, e.g. if you have different areas in your + website like "products" and "infos". + + ``Codappix\SearchCore\DataProcessing\RelationResolverProcessor`` + Resolves all relations using the TCA. + +Of course you are able to provide further processors. Just implement +``Codappix\SearchCore\DataProcessing\ProcessorInterface`` and use the FQCN (=Fully qualified +class name) as done in the examples above. + +By implementing also the same interface as necessary for TYPO3 +:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`, you are able to reuse the same code +also for Fluid to prepare the same record fetched from DB for your fluid. diff --git a/Documentation/source/configuration/searching.rst b/Documentation/source/configuration/searching.rst new file mode 100644 index 0000000..ae73fad --- /dev/null +++ b/Documentation/source/configuration/searching.rst @@ -0,0 +1,218 @@ +.. _configuration_options_search: + +Searching +========= + +.. _size: + +size +---- + +Used by: Elasticsearch connection while building search query. + +Defined how many search results should be fetched to be available in search result. + +Example:: + + plugin.tx_searchcore.settings.searching.size = 50 + +Default if not configured is 10. + +.. _facets: + +facets +------ + +Used by: Elasticsearch connection while building search query. + +Define aggregations for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-aggregations-bucket-terms-aggregation.html +Currently only the term facet is provided. + +Example:: + + plugin.tx_searchcore.settings.searching.facets { + contentTypes { + field = CType + } + } + +The above example will provide a facet with options for all found ``CType`` results together +with a count. + +.. _filter: + +filter +------ + +Used by: While building search request. + +Define filter that should be set for all requests. + +Example:: + + plugin.tx_searchcore.settings.searching.filter { + property = value + } + +For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``keyword``. + +.. _minimumShouldMatch: + +minimumShouldMatch +------------------ + +Used by: Elasticsearch connection while building search query. + +Define the minimum match for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-minimum-should-match.html + +Example:: + + plugin.tx_searchcore.settings.searching.minimumShouldMatch = 50% + +.. _boost: + +boost +----- + +Used by: Elasticsearch connection while building search query. + +Define fields that should boost the score for results. + +Example:: + + plugin.tx_searchcore.settings.searching.boost { + search_title = 3 + search_abstract = 1.5 + } + +For further information take a look at +https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_boosting_query_clauses.html + +.. _fieldValueFactor: + +fieldValueFactor +---------------- + +Used by: Elasticsearch connection while building search query. + +Define a field to use as a factor for scoring. The configuration is passed through to elastic +search ``field_value_factor``, see: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor + +Example:: + + plugin.tx_searchcore.settings.searching.field_value_factor { + field = rootlineLevel + modifier = reciprocal + factor = 2 + missing = 1 + } + +.. _mapping.filter: + +mapping.filter +-------------- + +Allows to configure filter more in depth. If a filter with the given key exists, the TypoScript will +be added. + +E.g. you submit a filter in form of: + +.. code-block:: html + + + + + +This will create a ``distance`` filter with subproperties. To make this filter actually work, you +can add the following TypoScript, which will be added to the filter:: + + mapping { + filter { + distance { + field = geo_distance + fields { + distance = distance + location = location + } + } + } + } + +``fields`` has a special meaning here. This will actually map the properties of the filter to fields +in elasticsearch. In above example they do match, but you can also use different names in your form. +On the left hand side is the elasticsearch field name, on the right side the one submitted as a +filter. + +The ``field``, in above example ``geo_distance``, will be used as the elasticsearch field for +filtering. This way you can use arbitrary filter names and map them to existing elasticsearch fields. + +.. _fields: + +fields +------ + +Defines the fields to fetch and search from elasticsearch. With the following sub keys: + +``query`` defines the fields to search in. Default is ``_all`` from 5.x times of elasticsearch. +Configure a comma separated list of fields to search in. This is necessary if you have configured +special mapping for some fields, or just want to search some fields. +The most hits get ranked highest. The following is an example configuration:: + + fields { + query = _all, city + } + +The following sub properties configure the fields to fetch from elasticsearch: + +First ``stored_fields`` which is a list of comma separated fields which actually exist and will be +added. Typically you will use ``_source`` to fetch the whole indexed fields. + +Second is ``script_fields``, which allow you to configure scripted fields for elasticsearch. +An example might look like the following:: + + fields { + script_fields { + distance { + condition = {request.filter.distance.location} + script { + params { + lat = {request.filter.distance.location.lat -> f:format.number()} + lon = {request.filter.distance.location.lon -> f:format.number()} + } + lang = painless + inline = doc["location"].arcDistance(params.lat,params.lon) * 0.001 + } + } + } + } + +In above example we add a single ``script_field`` called ``distance``. We add a condition when this +field should be added. The condition will be parsed as Fluidtemplate and is casted to bool via PHP. +If the condition is true, or no ``condition`` exists, the ``script_field`` will be added to the +query. The ``condition`` will be removed and everything else is submitted one to one to +elasticsearch, except each property is run through Fluidtemplate, to allow you to use information +from search request, e.g. to insert latitude and longitude from a filter, like in the above example. + +.. _sort: + +sort +---- + +Sort is handled like :ref:`fields`. + +.. _mode: + +mode +---- + +Used by: Controller while preparing action. + +Define to switch from search to filter mode. + +Example:: + + plugin.tx_searchcore.settings.searching { + mode = filter + } + +Only ``filter`` is allowed as value. Will submit an empty query to switch to filter mode. diff --git a/Documentation/source/features.rst b/Documentation/source/features.rst index 669e632..8baf453 100644 --- a/Documentation/source/features.rst +++ b/Documentation/source/features.rst @@ -15,6 +15,8 @@ configuration needs. Still it's possible to configure the indexer. Also custom classes can be used as indexers. +Furthermore a finisher for TYPO3 Form-Extension is provided to integrate indexing. + .. _features_search: Searching @@ -24,7 +26,7 @@ Currently all fields are searched for a single search input. Also multiple filter are supported. Filtering results by fields for string contents. -Even facets / aggregates are now possible. Therefore a mapping has to be defined in TypoScript for +Facets / aggregates are also possible. Therefore a mapping has to be defined in TypoScript for indexing, and the facets itself while searching. .. _features_planned: diff --git a/Documentation/source/indexer.rst b/Documentation/source/indexer.rst index ddc6772..3bb10bc 100644 --- a/Documentation/source/indexer.rst +++ b/Documentation/source/indexer.rst @@ -21,12 +21,18 @@ further stuff. The indexer is configurable through the following options: -* :ref:`allowedTables` - * :ref:`rootLineBlacklist` * :ref:`additionalWhereClause` +* :ref:`abstractFields` + +* :ref:`mapping` + +* :ref:`index` + +* :ref:`dataProcessing` + .. _PagesIndexer: PagesIndexer @@ -42,14 +48,18 @@ improve search. The indexer is configurable through the following options: -* :ref:`allowedTables` - * :ref:`rootLineBlacklist` * :ref:`additionalWhereClause` * :ref:`abstractFields` +* :ref:`mapping` + +* :ref:`index` + +* :ref:`dataProcessing` + .. note:: Not all relations are resolved yet, see :issue:`17` and :pr:`20`. diff --git a/Documentation/source/usage.rst b/Documentation/source/usage.rst index 1ca50d2..d57d643 100644 --- a/Documentation/source/usage.rst +++ b/Documentation/source/usage.rst @@ -18,6 +18,19 @@ This will index the table ``tt_content`` using the :ref:`TcaIndexer`. Only one index per call is available, to run multiple indexers, just make multiple calls. The indexers have to be defined in TypoScript via :ref:`configuration_options_index`. +.. _usage_manual_deletion: + +Manual deletion +--------------- + +You can trigger deletion for a single index from CLI:: + + ./typo3/cli_dispatch.phpsh extbase index:delete --identifier 'tt_content' + +This will delete the index for the table ``tt_content``. + +Only one delete per call is available, to run multiple deletions, just make multiple calls. + .. _usage_auto_indexing: Auto indexing @@ -30,6 +43,34 @@ The tables have to be configured via :ref:`configuration_options_index`. Not all hook operations are supported yet, see :issue:`27`. +.. _usage_form_finisher: + +Form finisher +------------- + +A form finisher is provided to integrate indexing into form extension. + +Add form finisher to your available finishers and configure it like: + +.. code-block:: yaml + :linenos: + + - + identifier: SearchCoreIndexer + options: + action: 'delete' + indexIdentifier: 'fe_users' + recordUid: '{FeUser.user.uid}' + +All three options are necessary, where + +``action`` + Is one of ``delete``, ``update`` or ``add``. +``indexIdentifier`` + Is a configured index identifier. +``recordUid`` + Has to be the uid of the record to index. + .. _usage_searching: Searching / Frontend Plugin diff --git a/Tests/Functional/Connection/Elasticsearch/IndexDeletionTest.php b/Tests/Functional/Connection/Elasticsearch/IndexDeletionTest.php new file mode 100644 index 0000000..8297a2a --- /dev/null +++ b/Tests/Functional/Connection/Elasticsearch/IndexDeletionTest.php @@ -0,0 +1,50 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Domain\Index\IndexerFactory; +use TYPO3\CMS\Extbase\Object\ObjectManager; + +class IndexDeletionTest extends AbstractFunctionalTestCase +{ + /** + * @test + */ + public function indexIsDeleted() + { + $this->client->getIndex('typo3content')->create(); + $this->assertTrue( + $this->client->getIndex('typo3content')->exists(), + 'Could not create index for test.' + ); + + \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) + ->get(IndexerFactory::class) + ->getIndexer('tt_content') + ->delete() + ; + + $this->assertFalse( + $this->client->getIndex('typo3content')->exists(), + 'Index could not be deleted through command controller.' + ); + } +} diff --git a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php index 4267d79..db446f3 100644 --- a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php +++ b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php @@ -206,6 +206,33 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase ); } + /** + * @test + */ + public function indexingDeltedRecordIfRecordShouldBeIndexedButIsNoLongerAvailableAndWasAlreadyIndexed() + { + \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) + ->get(IndexerFactory::class) + ->getIndexer('tt_content') + ->indexAllDocuments() + ; + + $response = $this->client->request('typo3content/_search?q=*:*'); + $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.'); + + $this->getDatabaseConnection() + ->exec_UPDATEquery('tt_content', 'uid = 10', ['hidden' => 1]); + + \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) + ->get(IndexerFactory::class) + ->getIndexer('tt_content') + ->indexDocument(10) + ; + + $response = $this->client->request('typo3content/_search?q=*:*'); + $this->assertSame($response->getData()['hits']['total'], 1, 'Not exactly 1 document is in index.'); + } + /** * @test */ diff --git a/Tests/Functional/Fixtures/BasicSetup.ts b/Tests/Functional/Fixtures/BasicSetup.ts index 9795b0f..3551670 100644 --- a/Tests/Functional/Fixtures/BasicSetup.ts +++ b/Tests/Functional/Fixtures/BasicSetup.ts @@ -43,6 +43,10 @@ plugin { field = CType } } + + fields { + query = _all + } } } } diff --git a/Tests/Unit/AbstractUnitTestCase.php b/Tests/Unit/AbstractUnitTestCase.php index c11e74e..c21209d 100644 --- a/Tests/Unit/AbstractUnitTestCase.php +++ b/Tests/Unit/AbstractUnitTestCase.php @@ -21,9 +21,40 @@ namespace Codappix\SearchCore\Tests\Unit; */ use TYPO3\CMS\Core\Tests\UnitTestCase as CoreTestCase; +use TYPO3\CMS\Core\Utility\GeneralUtility; +use TYPO3\CMS\Extbase\Object\ObjectManager; +use TYPO3\CMS\Form\Service\TranslationService; abstract class AbstractUnitTestCase extends CoreTestCase { + /** + * @var array A backup of registered singleton instances + */ + protected $singletonInstances = []; + + public function setUp() + { + parent::setUp(); + + $this->singletonInstances = GeneralUtility::getSingletonInstances(); + + // Disable caching backends to make TYPO3 parts work in unit test mode. + + \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance( + \TYPO3\CMS\Core\Cache\CacheManager::class + )->setCacheConfigurations([ + 'extbase_object' => [ + 'backend' => \TYPO3\CMS\Core\Cache\Backend\NullBackend::class, + ], + ]); + } + + public function tearDown() + { + GeneralUtility::resetSingletonInstances($this->singletonInstances); + parent::tearDown(); + } + /** * @return \TYPO3\CMS\Core\Log\LogManager */ @@ -43,4 +74,25 @@ abstract class AbstractUnitTestCase extends CoreTestCase return $logger; } + + /** + * Configure translation service mock for Form Finisher. + * + * This way parseOption will always return the configured value. + */ + protected function configureMockedTranslationService() + { + $translationService = $this->getMockBuilder(TranslationService::class)->getMock(); + $translationService->expects($this->any()) + ->method('translateFinisherOption') + ->willReturnCallback(function ($formRuntime, $finisherIdentifier, $optionKey, $optionValue) { + return $optionValue; + }); + $objectManager = $this->getMockBuilder(ObjectManager::class)->getMock(); + $objectManager->expects($this->any()) + ->method('get') + ->with(TranslationService::class) + ->willReturn($translationService); + GeneralUtility::setSingletonInstance(ObjectManager::class, $objectManager); + } } diff --git a/Tests/Unit/Command/IndexCommandControllerTest.php b/Tests/Unit/Command/IndexCommandControllerTest.php index 4c81672..9dcc7f3 100644 --- a/Tests/Unit/Command/IndexCommandControllerTest.php +++ b/Tests/Unit/Command/IndexCommandControllerTest.php @@ -91,4 +91,41 @@ class IndexCommandControllerTest extends AbstractUnitTestCase $this->subject->indexCommand('allowedTable'); } + + /** + * @test + */ + public function deletionIsPossible() + { + $indexerMock = $this->getMockBuilder(TcaIndexer::class) + ->disableOriginalConstructor() + ->getMock(); + $this->subject->expects($this->once()) + ->method('outputLine') + ->with('allowedTable was deleted.'); + $this->indexerFactory->expects($this->once()) + ->method('getIndexer') + ->with('allowedTable') + ->will($this->returnValue($indexerMock)); + + $indexerMock->expects($this->once()) + ->method('delete'); + $this->subject->deleteCommand('allowedTable'); + } + + /** + * @test + */ + public function deletionForNonExistingIndexerDoesNotWork() + { + $this->subject->expects($this->once()) + ->method('outputLine') + ->with('No indexer found for: nonAllowedTable'); + $this->indexerFactory->expects($this->once()) + ->method('getIndexer') + ->with('nonAllowedTable') + ->will($this->throwException(new NoMatchingIndexerException)); + + $this->subject->deleteCommand('nonAllowedTable'); + } } diff --git a/Tests/Unit/Configuration/ConfigurationUtilityTest.php b/Tests/Unit/Configuration/ConfigurationUtilityTest.php new file mode 100644 index 0000000..b2dc5f8 --- /dev/null +++ b/Tests/Unit/Configuration/ConfigurationUtilityTest.php @@ -0,0 +1,143 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Configuration\ConfigurationUtility; +use Codappix\SearchCore\Connection\SearchRequestInterface; +use Codappix\SearchCore\Domain\Model\SearchRequest; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class ConfigurationUtilityTest extends AbstractUnitTestCase +{ + /** + * @test + * @dataProvider possibleRequestAndConfigurationForFluidtemplate + */ + public function recursiveEntriesAreProcessedAsFluidtemplate(SearchRequestInterface $searchRequest, array $array, array $expected) + { + $subject = new ConfigurationUtility(); + + $this->assertSame( + $expected, + $subject->replaceArrayValuesWithRequestContent($searchRequest, $array), + 'Entries in array were not parsed as fluid template with search request.' + ); + } + + public function possibleRequestAndConfigurationForFluidtemplate() : array + { + return [ + 'Nothing in array' => [ + 'searchRequest' => new SearchRequest(), + 'array' => [], + 'expected' => [], + ], + 'Small array with nothing to replace' => [ + 'searchRequest' => new SearchRequest(), + 'array' => [ + 'key1' => 'value1', + ], + 'expected' => [ + 'key1' => 'value1', + ], + ], + 'Rescursive array with replacements' => [ + 'searchRequest' => call_user_func(function () { + $request = new SearchRequest(); + $request->setFilter([ + 'distance' => [ + 'location' => '10', + ], + ]); + return $request; + }), + 'array' => [ + 'sub1' => [ + 'sub1.1' => '{request.filter.distance.location}', + 'sub1.2' => '{request.nonExisting}', + ], + ], + 'expected' => [ + 'sub1' => [ + // Numberics are casted to double + 'sub1.1' => 10.0, + 'sub1.2' => null, + ], + ], + ], + ]; + } + + /** + * @test + * @dataProvider possibleConditionEntries + */ + public function conditionsAreHandledAsExpected(array $entries, array $expected) + { + $subject = new ConfigurationUtility(); + + $this->assertSame( + $expected, + $subject->filterByCondition($entries), + 'Conditions were not processed as expected.' + ); + } + + public function possibleConditionEntries() : array + { + return [ + 'Nothing in array' => [ + 'entries' => [], + 'expected' => [], + ], + 'Entries without condition' => [ + 'entries' => [ + 'key1' => 'value1', + ], + 'expected' => [ + 'key1' => 'value1', + ], + ], + 'Entry with matching condition' => [ + 'entries' => [ + 'sub1' => [ + 'condition' => true, + 'sub1.2' => 'something', + ], + ], + 'expected' => [ + 'sub1' => [ + 'sub1.2' => 'something', + ], + ], + ], + 'Entry with non matching condition' => [ + 'entries' => [ + 'sub1' => [ + 'condition' => false, + 'sub1.2' => 'something', + ], + ], + 'expected' => [], + ], + ]; + } +} diff --git a/Tests/Unit/DataProcessing/CopyToProcessorTest.php b/Tests/Unit/DataProcessing/CopyToProcessorTest.php new file mode 100644 index 0000000..2f9e498 --- /dev/null +++ b/Tests/Unit/DataProcessing/CopyToProcessorTest.php @@ -0,0 +1,85 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\DataProcessing\CopyToProcessor; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class CopyToProcessorTest extends AbstractUnitTestCase +{ + /** + * @test + * @dataProvider getPossibleRecordConfigurationCombinations + */ + public function fieldsAreCopiedAsConfigured(array $record, array $configuration, array $expectedRecord) + { + $subject = new CopyToProcessor(); + $processedRecord = $subject->processRecord($record, $configuration); + $this->assertSame( + $expectedRecord, + $processedRecord, + 'The processor did not return the expected processed record.' + ); + } + + /** + * @return array + */ + public function getPossibleRecordConfigurationCombinations() + { + return [ + 'Copy all fields to new field' => [ + 'record' => [ + 'field 1' => 'Some content like lorem', + 'field 2' => 'Some more content like ipsum', + ], + 'configuration' => [ + 'to' => 'new_field', + ], + 'expectedRecord' => [ + 'field 1' => 'Some content like lorem', + 'field 2' => 'Some more content like ipsum', + 'new_field' => 'Some content like lorem' . PHP_EOL . 'Some more content like ipsum', + ], + ], + 'Copy all fields with sub array to new field' => [ + 'record' => [ + 'field 1' => 'Some content like lorem', + 'field with sub2' => [ + 'Tag 1', + 'Tag 2', + ], + ], + 'configuration' => [ + 'to' => 'new_field', + ], + 'expectedRecord' => [ + 'field 1' => 'Some content like lorem', + 'field with sub2' => [ + 'Tag 1', + 'Tag 2', + ], + 'new_field' => 'Some content like lorem' . PHP_EOL . 'Tag 1' . PHP_EOL . 'Tag 2', + ], + ], + ]; + } +} diff --git a/Tests/Unit/DataProcessing/GeoPointProcessorTest.php b/Tests/Unit/DataProcessing/GeoPointProcessorTest.php new file mode 100644 index 0000000..99b8eb3 --- /dev/null +++ b/Tests/Unit/DataProcessing/GeoPointProcessorTest.php @@ -0,0 +1,113 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\DataProcessing\GeoPointProcessor; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class GeoPointProcessorTest extends AbstractUnitTestCase +{ + /** + * @test + * @dataProvider getPossibleRecordConfigurationCombinations + */ + public function geoPointsAreAddedAsConfigured(array $record, array $configuration, array $expectedRecord) + { + $subject = new GeoPointProcessor(); + $processedRecord = $subject->processRecord($record, $configuration); + $this->assertSame( + $expectedRecord, + $processedRecord, + 'The processor did not return the expected processed record.' + ); + } + + /** + * @return array + */ + public function getPossibleRecordConfigurationCombinations() + { + return [ + 'Create new field with existing lat and lng' => [ + 'record' => [ + 'lat' => 23.232, + 'lng' => 45.43, + ], + 'configuration' => [ + 'to' => 'location', + 'lat' => 'lat', + 'lon' => 'lng', + ], + 'expectedRecord' => [ + 'lat' => 23.232, + 'lng' => 45.43, + 'location' => [ + 'lat' => 23.232, + 'lon' => 45.43, + ], + ], + ], + 'Do not create new field due to missing configuration' => [ + 'record' => [ + 'lat' => 23.232, + 'lng' => 45.43, + ], + 'configuration' => [ + 'to' => 'location', + ], + 'expectedRecord' => [ + 'lat' => 23.232, + 'lng' => 45.43, + ], + ], + 'Do not create new field due to missing lat and lon' => [ + 'record' => [ + 'lat' => '', + 'lng' => '', + ], + 'configuration' => [ + 'to' => 'location', + 'lat' => 'lat', + 'lon' => 'lng', + ], + 'expectedRecord' => [ + 'lat' => '', + 'lng' => '', + ], + ], + 'Do not create new field due to invalid lat and lon' => [ + 'record' => [ + 'lat' => 'av', + 'lng' => 'dsf', + ], + 'configuration' => [ + 'to' => 'location', + 'lat' => 'lat', + 'lon' => 'lng', + ], + 'expectedRecord' => [ + 'lat' => 'av', + 'lng' => 'dsf', + ], + ], + ]; + } +} diff --git a/Tests/Unit/DataProcessing/RemoveProcessorTest.php b/Tests/Unit/DataProcessing/RemoveProcessorTest.php new file mode 100644 index 0000000..dc55f73 --- /dev/null +++ b/Tests/Unit/DataProcessing/RemoveProcessorTest.php @@ -0,0 +1,133 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\DataProcessing\RemoveProcessor; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class RemoveProcessorTest extends AbstractUnitTestCase +{ + /** + * @test + * @dataProvider getPossibleRecordConfigurationCombinations + */ + public function fieldsAreCopiedAsConfigured(array $record, array $configuration, array $expectedRecord) + { + $subject = new RemoveProcessor(); + $processedRecord = $subject->processRecord($record, $configuration); + $this->assertSame( + $expectedRecord, + $processedRecord, + 'The processor did not return the expected processed record.' + ); + } + + /** + * @return array + */ + public function getPossibleRecordConfigurationCombinations() + { + return [ + 'Nothing configured' => [ + 'record' => [ + 'field 1' => 'Some content like lorem', + 'field with sub2' => [ + 'Tag 1', + 'Tag 2', + ], + ], + 'configuration' => [ + ], + 'expectedRecord' => [ + 'field 1' => 'Some content like lorem', + 'field with sub2' => [ + 'Tag 1', + 'Tag 2', + ], + ], + ], + 'Single field configured' => [ + 'record' => [ + 'field 1' => 'Some content like lorem', + 'field with sub2' => [ + 'Tag 1', + 'Tag 2', + ], + ], + 'configuration' => [ + 'fields' => 'field with sub2', + '_typoScriptNodeValue' => 'Codappix\SearchCore\DataProcessing\RemoveProcessor', + ], + 'expectedRecord' => [ + 'field 1' => 'Some content like lorem', + ], + ], + 'Non existing field configured' => [ + 'record' => [ + 'field 1' => 'Some content like lorem', + 'field with sub2' => [ + 'Tag 1', + 'Tag 2', + ], + ], + 'configuration' => [ + 'fields' => 'non existing', + '_typoScriptNodeValue' => 'Codappix\SearchCore\DataProcessing\RemoveProcessor', + ], + 'expectedRecord' => [ + 'field 1' => 'Some content like lorem', + 'field with sub2' => [ + 'Tag 1', + 'Tag 2', + ], + ], + ], + 'Multiple fields configured' => [ + 'record' => [ + 'field 1' => 'Some content like lorem', + 'field with sub2' => [ + 'Tag 1', + 'Tag 2', + ], + 'field 3' => 'Some more like lorem', + ], + 'configuration' => [ + 'fields' => 'field 3, field with sub2', + '_typoScriptNodeValue' => 'Codappix\SearchCore\DataProcessing\RemoveProcessor', + ], + 'expectedRecord' => [ + 'field 1' => 'Some content like lorem', + ], + ], + 'Fields with "null" san be removed' => [ + 'record' => [ + 'field 1' => null, + ], + 'configuration' => [ + 'fields' => 'field 1', + '_typoScriptNodeValue' => 'Codappix\SearchCore\DataProcessing\RemoveProcessor', + ], + 'expectedRecord' => [ + ], + ], + ]; + } +} diff --git a/Tests/Unit/Domain/Index/AbstractIndexerTest.php b/Tests/Unit/Domain/Index/AbstractIndexerTest.php new file mode 100644 index 0000000..3bbc97d --- /dev/null +++ b/Tests/Unit/Domain/Index/AbstractIndexerTest.php @@ -0,0 +1,121 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Configuration\InvalidArgumentException; +use Codappix\SearchCore\Connection\ConnectionInterface; +use Codappix\SearchCore\DataProcessing\CopyToProcessor; +use Codappix\SearchCore\Domain\Index\AbstractIndexer; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class AbstractIndexerTest extends AbstractUnitTestCase +{ + /** + * @var TcaTableService + */ + protected $subject; + + /** + * @var ConfigurationContainerInterface + */ + protected $configuration; + + /** + * @var ConnectionInterface + */ + protected $connection; + + public function setUp() + { + parent::setUp(); + + $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock(); + $this->connection = $this->getMockBuilder(ConnectionInterface::class)->getMock(); + + $this->subject = $this->getMockForAbstractClass(AbstractIndexer::class, [ + $this->connection, + $this->configuration + ]); + $this->subject->injectLogger($this->getMockedLogger()); + $this->subject->setIdentifier('testTable'); + $this->subject->expects($this->any()) + ->method('getDocumentName') + ->willReturn('testTable'); + } + + /** + * @test + */ + public function executesConfiguredDataProcessingWithConfiguration() + { + $record = ['field 1' => 'test']; + $expectedRecord = $record; + $expectedRecord['new_test_field'] = 'test'; + $expectedRecord['new_test_field2'] = 'test' . PHP_EOL . 'test'; + $expectedRecord['search_abstract'] = ''; + + $this->configuration->expects($this->exactly(2)) + ->method('get') + ->withConsecutive(['indexing.testTable.dataProcessing'], ['indexing.testTable.abstractFields']) + ->will($this->onConsecutiveCalls([ + '1' => [ + '_typoScriptNodeValue' => CopyToProcessor::class, + 'to' => 'new_test_field', + ], + '2' => [ + '_typoScriptNodeValue' => CopyToProcessor::class, + 'to' => 'new_test_field2', + ], + ], $this->throwException(new InvalidArgumentException))); + $this->subject->expects($this->once()) + ->method('getRecord') + ->with(1) + ->willReturn($record) + ; + + $this->connection->expects($this->once())->method('addDocument')->with('testTable', $expectedRecord); + $this->subject->indexDocument(1); + } + + /** + * @test + */ + public function executesNoDataProcessingForMissingConfiguration() + { + $record = ['field 1' => 'test']; + $expectedRecord = $record; + $expectedRecord['search_abstract'] = ''; + + $this->configuration->expects($this->exactly(2)) + ->method('get') + ->withConsecutive(['indexing.testTable.dataProcessing'], ['indexing.testTable.abstractFields']) + ->will($this->throwException(new InvalidArgumentException)); + $this->subject->expects($this->once()) + ->method('getRecord') + ->with(1) + ->willReturn($record) + ; + + $this->connection->expects($this->once())->method('addDocument')->with('testTable', $expectedRecord); + $this->subject->indexDocument(1); + } +} diff --git a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php index 9e88d4b..3778af9 100644 --- a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php +++ b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php @@ -21,6 +21,8 @@ namespace Codappix\SearchCore\Tests\Unit\Domain\Index\TcaIndexer; */ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\DataProcessing\CopyToProcessor; +use Codappix\SearchCore\Domain\Index\TcaIndexer\RelationResolver; use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService; use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; @@ -82,4 +84,48 @@ class TcaTableServiceTest extends AbstractUnitTestCase $this->subject->getWhereClause() ); } + + /** + * @test + */ + public function allConfiguredAndAllowedTcaColumnsAreReturnedAsFields() + { + $GLOBALS['TCA']['test_table'] = [ + 'ctrl' => [ + 'languageField' => 'sys_language', + ], + 'columns' => [ + 'sys_language' => [], + 't3ver_oid' => [], + 'available_column' => [ + 'config' => [ + 'type' => 'input', + ], + ], + 'user_column' => [ + 'config' => [ + 'type' => 'user', + ], + ], + 'passthrough_column' => [ + 'config' => [ + 'type' => 'passthrough', + ], + ], + ], + ]; + $subject = new TcaTableService( + 'test_table', + $this->getMockBuilder(RelationResolver::class)->getMock(), + $this->configuration + ); + $this->inject($subject, 'logger', $this->getMockedLogger()); + + $this->assertSame( + 'test_table.uid,test_table.pid,test_table.available_column', + $subject->getFields(), + '' + ); + unset($GLOBALS['TCA']['test_table']); + } } diff --git a/Tests/Unit/Domain/Model/SearchRequestTest.php b/Tests/Unit/Domain/Model/SearchRequestTest.php new file mode 100644 index 0000000..127baf2 --- /dev/null +++ b/Tests/Unit/Domain/Model/SearchRequestTest.php @@ -0,0 +1,80 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Domain\Model\SearchRequest; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class SearchRequestTest extends AbstractUnitTestCase +{ + /** + * @test + * @dataProvider possibleEmptyFilter + */ + public function emptyFilterWillNotBeSet(array $filter) + { + $searchRequest = new SearchRequest(); + $searchRequest->setFilter($filter); + + $this->assertSame( + [], + $searchRequest->getFilter(), + 'Empty filter were set, even if they should not.' + ); + } + + public function possibleEmptyFilter() + { + return [ + 'Complete empty Filter' => [ + 'filter' => [], + ], + 'Single filter with empty value' => [ + 'filter' => [ + 'someFilter' => '', + ], + ], + 'Single filter with empty recursive values' => [ + 'filter' => [ + 'someFilter' => [ + 'someKey' => '', + ], + ], + ], + ]; + } + + /** + * @test + */ + public function filterIsSet() + { + $filter = ['someField' => 'someValue']; + $searchRequest = new SearchRequest(); + $searchRequest->setFilter($filter); + + $this->assertSame( + $filter, + $searchRequest->getFilter(), + 'Filter was not set.' + ); + } +} diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index f719fdf..cf74ce3 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -21,6 +21,7 @@ namespace Codappix\SearchCore\Tests\Unit\Domain\Search; */ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Configuration\ConfigurationUtility; use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Domain\Model\FacetRequest; use Codappix\SearchCore\Domain\Model\SearchRequest; @@ -44,7 +45,8 @@ class QueryFactoryTest extends AbstractUnitTestCase parent::setUp(); $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock(); - $this->subject = new QueryFactory($this->getMockedLogger(), $this->configuration); + $configurationUtility = new ConfigurationUtility(); + $this->subject = new QueryFactory($this->getMockedLogger(), $this->configuration, $configurationUtility); } /** @@ -52,10 +54,10 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function creationOfQueryWorksInGeneral() { - $this->mockConfiguration(); - $searchRequest = new SearchRequest('SearchWord'); + $this->configureConfigurationMockWithDefault(); + $query = $this->subject->create($searchRequest); $this->assertInstanceOf( \Elastica\Query::class, @@ -69,7 +71,7 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function filterIsAddedToQuery() { - $this->mockConfiguration(); + $this->configureConfigurationMockWithDefault(); $searchRequest = new SearchRequest('SearchWord'); $searchRequest->setFilter(['field' => 'content']); @@ -79,7 +81,7 @@ class QueryFactoryTest extends AbstractUnitTestCase [ ['term' => ['field' => 'content']] ], - $query->toArray()['query']['function_score']['query']['bool']['filter'], + $query->toArray()['query']['bool']['filter'], 'Filter was not added to query.' ); } @@ -89,13 +91,11 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function emptyFilterIsNotAddedToQuery() { - $this->mockConfiguration(); + $this->configureConfigurationMockWithDefault(); $searchRequest = new SearchRequest('SearchWord'); $searchRequest->setFilter([ 'field' => '', - 'field1' => 0, - 'field2' => false, ]); $this->assertFalse( @@ -116,8 +116,7 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function facetsAreAddedToQuery() { - $this->mockConfiguration(); - + $this->configureConfigurationMockWithDefault(); $searchRequest = new SearchRequest('SearchWord'); $searchRequest->addFacet(new FacetRequest('Identifier', 'FieldName')); $searchRequest->addFacet(new FacetRequest('Identifier 2', 'FieldName 2')); @@ -146,8 +145,7 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function sizeIsAddedToQuery() { - $this->mockConfiguration(); - + $this->configureConfigurationMockWithDefault(); $searchRequest = new SearchRequest('SearchWord'); $searchRequest->setLimit(45); $searchRequest->setOffset(35); @@ -170,9 +168,8 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function searchTermIsAddedToQuery() { - $this->mockConfiguration(); - $searchRequest = new SearchRequest('SearchWord'); + $this->configureConfigurationMockWithDefault(); $query = $this->subject->create($searchRequest); $this->assertSame( @@ -191,7 +188,7 @@ class QueryFactoryTest extends AbstractUnitTestCase ], ], ], - $query->toArray()['query']['function_score']['query'], + $query->toArray()['query'], 'Search term was not added to query as expected.' ); } @@ -201,28 +198,38 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function minimumShouldMatchIsAddedToQuery() { - $this->configuration->expects($this->once()) - ->method('getIfExists') - ->with('searching.minimumShouldMatch') - ->willReturn('50%'); - $this->mockConfiguration(); - $searchRequest = new SearchRequest('SearchWord'); + $this->configuration->expects($this->any()) + ->method('getIfExists') + ->withConsecutive( + ['searching.minimumShouldMatch'], + ['searching.sort'] + ) + ->will($this->onConsecutiveCalls( + '50%', + null + )); + $this->configureConfigurationMockWithDefault(); $query = $this->subject->create($searchRequest); - $this->assertArraySubset( + $this->assertSame( [ 'bool' => [ 'must' => [ [ 'multi_match' => [ + 'type' => 'most_fields', + 'query' => 'SearchWord', + 'fields' => [ + 'test_field', + ], 'minimum_should_match' => '50%', ], ], ], ], ], - $query->toArray()['query']['function_score']['query'], + $query->toArray()['query'], 'minimum_should_match was not added to query as configured.' ); } @@ -234,15 +241,23 @@ class QueryFactoryTest extends AbstractUnitTestCase { $searchRequest = new SearchRequest('SearchWord'); - $this->configuration->expects($this->exactly(3)) + $this->configuration->expects($this->any()) ->method('get') - ->withConsecutive(['searching.fields'], ['searching.boost'], ['searching.fieldValueFactor']) + ->withConsecutive( + ['searching.fields.query'], + ['searching.boost'], + ['searching.fields.stored_fields'], + ['searching.fields.script_fields'], + ['searching.fieldValueFactor'] + ) ->will($this->onConsecutiveCalls( 'test_field', [ 'search_title' => 3, 'search_abstract' => 1.5, ], + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), $this->throwException(new InvalidArgumentException) )); @@ -283,12 +298,20 @@ class QueryFactoryTest extends AbstractUnitTestCase 'factor' => '2', 'missing' => '1', ]; - $this->configuration->expects($this->exactly(3)) + $this->configuration->expects($this->any()) ->method('get') - ->withConsecutive(['searching.fields'], ['searching.boost'], ['searching.fieldValueFactor']) + ->withConsecutive( + ['searching.fields.query'], + ['searching.boost'], + ['searching.fields.stored_fields'], + ['searching.fields.script_fields'], + ['searching.fieldValueFactor'] + ) ->will($this->onConsecutiveCalls( 'test_field', $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), $fieldConfig )); @@ -324,27 +347,279 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function emptySearchStringWillNotAddSearchToQuery() { - $this->mockConfiguration(); - $searchRequest = new SearchRequest(); + $this->configureConfigurationMockWithDefault(); + $query = $this->subject->create($searchRequest); - $this->assertNull( - $query->toArray()['query']['function_score']['query'], + $this->assertInstanceOf( + stdClass, + $query->toArray()['query']['match_all'], 'Empty search request does not create expected query.' ); } - protected function mockConfiguration() + /** + * @test + */ + public function configuredQueryFieldsAreAddedToQuery() + { + $searchRequest = new SearchRequest('SearchWord'); + + $this->configuration->expects($this->any()) + ->method('get') + ->withConsecutive( + ['searching.fields.query'], + ['searching.boost'], + ['searching.fields.stored_fields'], + ['searching.fields.script_fields'], + ['searching.fieldValueFactor'] + ) + ->will($this->onConsecutiveCalls( + 'test_field, field1, field2', + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException) + )); + + $query = $this->subject->create($searchRequest); + $this->assertArraySubset( + [ + 'bool' => [ + 'must' => [ + [ + 'multi_match' => [ + 'type' => 'most_fields', + 'query' => 'SearchWord', + 'fields' => [ + 'test_field', + 'field1', + 'field2', + ], + ], + ], + ], + ], + ], + $query->toArray()['query'], + 'Configured fields were not added to query as configured.' + ); + } + + /** + * @test + */ + public function storedFieldsAreAddedToQuery() + { + $searchRequest = new SearchRequest(); + + $this->configuration->expects($this->any()) + ->method('get') + ->withConsecutive( + ['searching.boost'], + ['searching.fields.stored_fields'], + ['searching.fields.script_fields'], + ['searching.fieldValueFactor'] + ) + ->will($this->onConsecutiveCalls( + $this->throwException(new InvalidArgumentException), + '_source, something,nothing', + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException) + )); + + $query = $this->subject->create($searchRequest); + $this->assertSame( + ['_source', 'something', 'nothing'], + $query->toArray()['stored_fields'], + 'Stored fields were not added to query as expected.' + ); + } + + /** + * @test + */ + public function storedFieldsAreNotAddedToQuery() + { + $searchRequest = new SearchRequest(); + + $this->configuration->expects($this->any()) + ->method('get') + ->withConsecutive( + ['searching.boost'], + ['searching.fields.stored_fields'], + ['searching.fields.script_fields'], + ['searching.fieldValueFactor'] + ) + ->will($this->onConsecutiveCalls( + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException) + )); + + $query = $this->subject->create($searchRequest); + $this->assertFalse( + isset($query->toArray()['stored_fields']), + 'Stored fields were added to query even if not configured.' + ); + } + + /** + * @test + */ + public function scriptFieldsAreAddedToQuery() + { + $searchRequest = new SearchRequest('query value'); + + $this->configuration->expects($this->any()) + ->method('get') + ->withConsecutive( + ['searching.fields.query'], + ['searching.boost'], + ['searching.fields.stored_fields'], + ['searching.fields.script_fields'], + ['searching.fieldValueFactor'] + ) + ->will($this->onConsecutiveCalls( + 'test_field', + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), + [ + 'field1' => [ + 'config' => 'something', + ], + 'field2' => [ + 'config' => '{request.query}', + ], + ], + $this->throwException(new InvalidArgumentException) + )); + + $query = $this->subject->create($searchRequest); + $this->assertSame( + [ + 'field1' => [ + 'config' => 'something', + ], + 'field2' => [ + 'config' => 'query value', + ], + ], + $query->toArray()['script_fields'], + 'Script fields were not added to query as expected.' + ); + } + + /** + * @test + */ + public function scriptFieldsAreNotAddedToQuery() + { + $searchRequest = new SearchRequest(); + + $this->configuration->expects($this->any()) + ->method('get') + ->withConsecutive( + ['searching.boost'], + ['searching.fields.stored_fields'], + ['searching.fields.script_fields'], + ['searching.fieldValueFactor'] + ) + ->will($this->onConsecutiveCalls( + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException) + )); + + $query = $this->subject->create($searchRequest); + $this->assertTrue( + !isset($query->toArray()['script_fields']), + 'Script fields were added to query even if not configured.' + ); + } + + /** + * @test + */ + public function sortIsAddedToQuery() + { + $searchRequest = new SearchRequest('query value'); + + $this->configuration->expects($this->any()) + ->method('getIfExists') + ->withConsecutive( + ['searching.minimumShouldMatch'], + ['searching.sort'] + ) + ->will($this->onConsecutiveCalls( + null, + [ + 'field1' => [ + 'config' => 'something', + ], + 'field2' => [ + 'config' => '{request.query}', + ], + ] + )); + + $this->configureConfigurationMockWithDefault(); + + $query = $this->subject->create($searchRequest); + $this->assertSame( + [ + 'field1' => [ + 'config' => 'something', + ], + 'field2' => [ + 'config' => 'query value', + ], + ], + $query->toArray()['sort'], + 'Sort was not added to query as expected.' + ); + } + + /** + * @test + */ + public function sortIsNotAddedToQuery() + { + $searchRequest = new SearchRequest('query value'); + + $this->configuration->expects($this->any()) + ->method('getIfExists') + ->withConsecutive( + ['searching.minimumShouldMatch'], + ['searching.sort'] + ) + ->will($this->onConsecutiveCalls( + null, + null + )); + + $this->configureConfigurationMockWithDefault(); + + $query = $this->subject->create($searchRequest); + $this->assertTrue( + !isset($query->toArray()['sort']), + 'Sort was added to query even if not configured.' + ); + } + + protected function configureConfigurationMockWithDefault() { $this->configuration->expects($this->any()) ->method('get') - ->will($this->returnCallback(function ($option) { - if ($option === 'searching.fields') { + ->will($this->returnCallback(function ($configName) { + if ($configName === 'searching.fields.query') { return 'test_field'; } - return $this->throwException(new InvalidArgumentException); + throw new InvalidArgumentException(); })); } } diff --git a/Tests/Unit/Domain/Search/SearchServiceTest.php b/Tests/Unit/Domain/Search/SearchServiceTest.php index 9149e51..c63e29e 100644 --- a/Tests/Unit/Domain/Search/SearchServiceTest.php +++ b/Tests/Unit/Domain/Search/SearchServiceTest.php @@ -178,4 +178,29 @@ class SearchServiceTest extends AbstractUnitTestCase $searchRequest->setFilter(['anotherProperty' => 'anything']); $this->subject->search($searchRequest); } + + /** + * @test + */ + public function emptyConfiguredFilterIsNotChangingRequestWithExistingFilter() + { + $this->configuration->expects($this->exactly(2)) + ->method('getIfExists') + ->withConsecutive(['searching.size'], ['searching.facets']) + ->will($this->onConsecutiveCalls(null, null)); + $this->configuration->expects($this->exactly(1)) + ->method('get') + ->with('searching.filter') + ->willReturn(['anotherProperty' => '']); + + $this->connection->expects($this->once()) + ->method('search') + ->with($this->callback(function ($searchRequest) { + return $searchRequest->getFilter() === ['anotherProperty' => 'anything']; + })); + + $searchRequest = new SearchRequest('SearchWord'); + $searchRequest->setFilter(['anotherProperty' => 'anything']); + $this->subject->search($searchRequest); + } } diff --git a/composer.json b/composer.json index e4c182b..96dc524 100644 --- a/composer.json +++ b/composer.json @@ -16,7 +16,7 @@ } }, "require" : { - "php": ">=5.6.0", + "php": ">=7.0", "typo3/cms": "~7.6", "ruflin/elastica": "~3.2" }, @@ -35,6 +35,9 @@ ] }, "extra": { + "branch-alias": { + "dev-develop": "1.0.x-dev" + }, "typo3/cms": { "cms-package-dir": "{$vendor-dir}/typo3/cms", "web-dir": ".Build/web" diff --git a/ext_emconf.php b/ext_emconf.php index 118e2df..30b7819 100644 --- a/ext_emconf.php +++ b/ext_emconf.php @@ -8,7 +8,7 @@ $EM_CONF[$_EXTKEY] = [ 'constraints' => [ 'depends' => [ 'typo3' => '7.6.0-7.6.99', - 'php' => '5.6.0-7.99.99' + 'php' => '7.0.0-7.99.99' ], 'conflicts' => [], ], diff --git a/ext_localconf.php b/ext_localconf.php index 882f6ad..7743804 100644 --- a/ext_localconf.php +++ b/ext_localconf.php @@ -33,7 +33,7 @@ call_user_func( 'Search' => 'search' ], [ - 'Search' => 'search' // TODO: Enable caching. But submitting form results in previous result?! + 'Search' => 'search' ] );