diff --git a/.travis.yml b/.travis.yml index 4490198..73a7888 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,8 @@ before_install: language: php php: + - 5.6 + - 7.0 - 7.1 env: @@ -23,6 +25,7 @@ env: - typo3DatabaseHost="127.0.0.1" - typo3DatabaseUsername="travis" - typo3DatabasePassword="" + - TYPO3_VERSION="~7.6" matrix: fast_finish: true diff --git a/Classes/Connection/Elasticsearch/IndexFactory.php b/Classes/Connection/Elasticsearch/IndexFactory.php index 66b448b..49956f0 100644 --- a/Classes/Connection/Elasticsearch/IndexFactory.php +++ b/Classes/Connection/Elasticsearch/IndexFactory.php @@ -39,6 +39,21 @@ class IndexFactory implements Singleton */ protected $configuration; + /** + * @var \TYPO3\CMS\Core\Log\Logger + */ + protected $logger; + + /** + * Inject log manager to get concrete logger from it. + * + * @param \TYPO3\CMS\Core\Log\LogManager $logManager + */ + public function injectLogger(\TYPO3\CMS\Core\Log\LogManager $logManager) + { + $this->logger = $logManager->getLogger(__CLASS__); + } + /** * @param ConfigurationContainerInterface $configuration */ @@ -60,7 +75,10 @@ class IndexFactory implements Singleton $index = $connection->getClient()->getIndex('typo3content'); if ($index->exists() === false) { - $index->create($this->getConfigurationFor($documentType)); + $config = $this->getConfigurationFor($documentType); + $this->logger->debug(sprintf('Create index %s.', $documentType), [$documentType, $config]); + $index->create($config); + $this->logger->debug(sprintf('Created index %s.', $documentType), [$documentType]); } return $index; @@ -76,9 +94,11 @@ class IndexFactory implements Singleton try { $configuration = $this->configuration->get('indexing.' . $documentType . '.index'); - if (isset($configuration['analysis']['analyzer'])) { - foreach ($configuration['analysis']['analyzer'] as $key => $analyzer) { - $configuration['analysis']['analyzer'][$key] = $this->prepareAnalyzerConfiguration($analyzer); + foreach (['analyzer', 'filter'] as $optionsToExpand) { + if (isset($configuration['analysis'][$optionsToExpand])) { + foreach ($configuration['analysis'][$optionsToExpand] as $key => $options) { + $configuration['analysis'][$optionsToExpand][$key] = $this->prepareOptions($options); + } } } @@ -89,20 +109,20 @@ class IndexFactory implements Singleton } /** - * @param array $analyzer + * @param array $options * * @return array */ - protected function prepareAnalyzerConfiguration(array $analyzer) + protected function prepareOptions(array $options) { - $fieldsToExplode = ['char_filter', 'filter']; + $fieldsToExplode = ['char_filter', 'filter', 'word_list']; foreach ($fieldsToExplode as $fieldToExplode) { - if (isset($analyzer[$fieldToExplode])) { - $analyzer[$fieldToExplode] = GeneralUtility::trimExplode(',', $analyzer[$fieldToExplode], true); + if (isset($options[$fieldToExplode])) { + $options[$fieldToExplode] = GeneralUtility::trimExplode(',', $options[$fieldToExplode], true); } } - return $analyzer; + return $options; } } diff --git a/Classes/Connection/Elasticsearch/SearchResult.php b/Classes/Connection/Elasticsearch/SearchResult.php index f45f02f..3919722 100644 --- a/Classes/Connection/Elasticsearch/SearchResult.php +++ b/Classes/Connection/Elasticsearch/SearchResult.php @@ -83,7 +83,7 @@ class SearchResult implements SearchResultInterface /** * Return all facets, if any. * - * @return array + * @return array */ public function getFacets() { diff --git a/Classes/Connection/SearchResultInterface.php b/Classes/Connection/SearchResultInterface.php index d52a8ba..60718cd 100644 --- a/Classes/Connection/SearchResultInterface.php +++ b/Classes/Connection/SearchResultInterface.php @@ -35,7 +35,7 @@ interface SearchResultInterface extends \Iterator, \Countable, QueryResultInterf /** * Return all facets, if any. * - * @return array + * @return array */ public function getFacets(); diff --git a/Classes/Database/Doctrine/Where.php b/Classes/DataProcessing/CopyToProcessor.php similarity index 53% rename from Classes/Database/Doctrine/Where.php rename to Classes/DataProcessing/CopyToProcessor.php index 6586b8a..e9eb8cf 100644 --- a/Classes/Database/Doctrine/Where.php +++ b/Classes/DataProcessing/CopyToProcessor.php @@ -1,5 +1,5 @@ @@ -20,31 +20,35 @@ namespace Codappix\SearchCore\Database\Doctrine; * 02110-1301, USA. */ -class Where +/** + * Copies values from one field to another one. + */ +class CopyToProcessor implements ProcessorInterface { - /** - * @var string - */ - protected $statement = ''; - - /** - * @var array - */ - protected $parameters = []; - - public function __construct(string $statement, array $parameters) + public function processRecord(array $record, array $configuration) { - $this->statement = $statement; - $this->parameters = $parameters; + $all = []; + + $this->addArray($all, $record); + $all = array_filter($all); + $record[$configuration['to']] = implode(PHP_EOL, $all); + + return $record; } - public function getStatement() : string + /** + * @param array &$to + * @param array $from + */ + protected function addArray(array &$to, array $from) { - return $this->statement; - } + foreach ($from as $property => $value) { + if (is_array($value)) { + $this->addArray($to, $value); + continue; + } - public function getParameters() : array - { - return $this->parameters; + $to[] = (string) $value; + } } } diff --git a/Classes/Database/Doctrine/Join.php b/Classes/DataProcessing/ProcessorInterface.php similarity index 62% rename from Classes/Database/Doctrine/Join.php rename to Classes/DataProcessing/ProcessorInterface.php index df1a8c6..841edb9 100644 --- a/Classes/Database/Doctrine/Join.php +++ b/Classes/DataProcessing/ProcessorInterface.php @@ -1,5 +1,5 @@ @@ -20,31 +20,20 @@ namespace Codappix\SearchCore\Database\Doctrine; * 02110-1301, USA. */ -class Join +/** + * All DataProcessing Processor should implement this interface, otherwise they + * will not be executed. + */ +interface ProcessorInterface { /** - * @var string + * Processes the given record. + * Also retrieves the configuration for this processor instance. + * + * @param array $record + * @param array $configuration + * + * @return array */ - protected $table = ''; - - /** - * @var string - */ - protected $condition = ''; - - public function __construct(string $table, string $condition) - { - $this->table = $table; - $this->condition = $condition; - } - - public function getTable() : string - { - return $this->table; - } - - public function getCondition() : string - { - return $this->condition; - } + public function processRecord(array $record, array $configuration); } diff --git a/Classes/Domain/Index/IndexerFactory.php b/Classes/Domain/Index/IndexerFactory.php index dbae818..e228780 100644 --- a/Classes/Domain/Index/IndexerFactory.php +++ b/Classes/Domain/Index/IndexerFactory.php @@ -22,8 +22,6 @@ namespace Codappix\SearchCore\Domain\Index; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\InvalidArgumentException; -use Codappix\SearchCore\Domain\Index\IndexerInterface; -use Codappix\SearchCore\Domain\Index\TcaIndexer; use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService; use TYPO3\CMS\Core\SingletonInterface as Singleton; use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; diff --git a/Classes/Domain/Index/TcaIndexer.php b/Classes/Domain/Index/TcaIndexer.php index c35b9ea..44d7c46 100644 --- a/Classes/Domain/Index/TcaIndexer.php +++ b/Classes/Domain/Index/TcaIndexer.php @@ -22,9 +22,6 @@ namespace Codappix\SearchCore\Domain\Index; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Connection\ConnectionInterface; -use TYPO3\CMS\Core\Database\ConnectionPool; -use TYPO3\CMS\Core\Database\Query\QueryBuilder; -use TYPO3\CMS\Core\Utility\GeneralUtility; /** * Will index the given table using configuration from TCA. @@ -58,12 +55,14 @@ class TcaIndexer extends AbstractIndexer */ protected function getRecords($offset, $limit) { - $records = $this->getQuery() - ->setFirstResult($offset) - ->setMaxResults($limit) - ->execute() - ->fetchAll(); - + $records = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows( + $this->tcaTableService->getFields(), + $this->tcaTableService->getTableClause(), + $this->tcaTableService->getWhereClause(), + '', + '', + (int) $offset . ',' . (int) $limit + ); if ($records === null) { return null; } @@ -83,9 +82,12 @@ class TcaIndexer extends AbstractIndexer */ protected function getRecord($identifier) { - $query = $this->getQuery(); - $query = $query->andWhere($this->tcaTableService->getTableName() . '.uid = ' . (int) $identifier); - $record = $query->execute()->fetch(); + $record = $GLOBALS['TYPO3_DB']->exec_SELECTgetSingleRow( + $this->tcaTableService->getFields(), + $this->tcaTableService->getTableClause(), + $this->tcaTableService->getWhereClause() + . ' AND ' . $this->tcaTableService->getTableName() . '.uid = ' . (int) $identifier + ); if ($record === false || $record === null) { throw new NoRecordFoundException( @@ -105,29 +107,4 @@ class TcaIndexer extends AbstractIndexer { return $this->tcaTableService->getTableName(); } - - protected function getQuery($tcaTableService = null) : QueryBuilder - { - if ($tcaTableService === null) { - $tcaTableService = $this->tcaTableService; - } - $queryBuilder = $this->getDatabaseConnection()->getQueryBuilderForTable($tcaTableService->getTableName()); - $where = $tcaTableService->getWhereClause(); - $query = $queryBuilder->select(... $tcaTableService->getFields()) - ->from($tcaTableService->getTableClause()) - ->where($where->getStatement()) - ->setParameters($where->getParameters()); - - foreach ($tcaTableService->getJoins() as $join) { - $query->from($join->getTable()); - $query->andWhere($join->getCondition()); - } - - return $query; - } - - protected function getDatabaseConnection() - { - return GeneralUtility::makeInstance(ConnectionPool::class); - } } diff --git a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php index b396b73..558af09 100644 --- a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php +++ b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php @@ -34,6 +34,12 @@ class PagesIndexer extends TcaIndexer */ protected $contentTableService; + /** + * @var \TYPO3\CMS\Core\Resource\FileRepository + * @inject + */ + protected $fileRepository; + /** * @param TcaTableService $tcaTableService * @param TcaTableService $tcaTableService @@ -65,32 +71,69 @@ class PagesIndexer extends TcaIndexer } } - $record['content'] = $this->fetchContentForPage($record['uid']); + $content = $this->fetchContentForPage($record['uid']); + if ($content !== []) { + $record['content'] = $content['content']; + $record['media'] = array_unique(array_merge($record['media'], $content['images'])); + } parent::prepareRecord($record); } /** * @param int $uid - * @return string + * @return [] */ protected function fetchContentForPage($uid) { - $contentElements = $this->getQuery($this->contentTableService)->execute()->fetchAll(); + $contentElements = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows( + $this->contentTableService->getFields(), + $this->contentTableService->getTableClause(), + $this->contentTableService->getWhereClause() . + sprintf(' AND %s.pid = %u', $this->contentTableService->getTableName(), $uid) + ); if ($contentElements === null) { $this->logger->debug('No content for page ' . $uid); - return ''; + return []; } $this->logger->debug('Fetched content for page ' . $uid); + $images = []; $content = []; foreach ($contentElements as $contentElement) { + $images = array_merge( + $images, + $this->getContentElementImages($contentElement['uid']) + ); $content[] = $contentElement['bodytext']; } - // Remove Tags. - // Interpret escaped new lines and special chars. - // Trim, e.g. trailing or leading new lines. - return trim(stripcslashes(strip_tags(implode(' ', $content)))); + return [ + // Remove Tags. + // Interpret escaped new lines and special chars. + // Trim, e.g. trailing or leading new lines. + 'content' => trim(stripcslashes(strip_tags(implode(' ', $content)))), + 'images' => $images, + ]; + } + + /** + * @param int $uidOfContentElement + * @return array + */ + protected function getContentElementImages($uidOfContentElement) + { + $imageRelationUids = []; + $imageRelations = $this->fileRepository->findByRelation( + 'tt_content', + 'image', + $uidOfContentElement + ); + + foreach ($imageRelations as $relation) { + $imageRelationUids[] = $relation->getUid(); + } + + return $imageRelationUids; } } diff --git a/Classes/Domain/Index/TcaIndexer/RelationResolver.php b/Classes/Domain/Index/TcaIndexer/RelationResolver.php index 88aa982..b6fc287 100644 --- a/Classes/Domain/Index/TcaIndexer/RelationResolver.php +++ b/Classes/Domain/Index/TcaIndexer/RelationResolver.php @@ -22,7 +22,6 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer; use TYPO3\CMS\Backend\Utility\BackendUtility; use TYPO3\CMS\Core\SingletonInterface as Singleton; -use TYPO3\CMS\Core\Utility\GeneralUtility; /** * Resolves relations from TCA using TCA. @@ -32,7 +31,7 @@ use TYPO3\CMS\Core\Utility\GeneralUtility; */ class RelationResolver implements Singleton { - public function resolveRelationsForRecord(TcaTableService $service, array &$record) : void + public function resolveRelationsForRecord(TcaTableService $service, array &$record) { foreach (array_keys($record) as $column) { // TODO: Define / configure fields to exclude?! @@ -76,7 +75,7 @@ class RelationResolver implements Singleton return []; } - protected function isRelation(array &$config) : bool + protected function isRelation(array &$config) { return isset($config['foreign_table']) || (isset($config['renderType']) && $config['renderType'] !== 'selectSingle') @@ -84,12 +83,12 @@ class RelationResolver implements Singleton ; } - protected function resolveForeignDbValue(string $value) : array + protected function resolveForeignDbValue($value) { return array_map('trim', explode(';', $value)); } - protected function resolveInlineValue(string $value) : array + protected function resolveInlineValue($value) { return array_map('trim', explode(',', $value)); } diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index dae86aa..e68a94f 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -21,8 +21,8 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer; */ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; -use Codappix\SearchCore\Database\Doctrine\Join; -use Codappix\SearchCore\Database\Doctrine\Where; +use Codappix\SearchCore\Configuration\InvalidArgumentException as InvalidConfigurationArgumentException; +use Codappix\SearchCore\DataProcessing\ProcessorInterface; use Codappix\SearchCore\Domain\Index\IndexingException; use TYPO3\CMS\Backend\Utility\BackendUtility; use TYPO3\CMS\Core\Utility\GeneralUtility; @@ -109,7 +109,7 @@ class TcaTableService /** * @return string */ - public function getTableName() : string + public function getTableName() { return $this->tableName; } @@ -117,9 +117,13 @@ class TcaTableService /** * @return string */ - public function getTableClause() : string + public function getTableClause() { - return $this->tableName; + if ($this->tableName === 'pages') { + return $this->tableName; + } + + return $this->tableName . ' LEFT JOIN pages on ' . $this->tableName . '.pid = pages.uid'; } /** @@ -128,7 +132,7 @@ class TcaTableService * @param array &$records * @return void */ - public function filterRecordsByRootLineBlacklist(array &$records) : void + public function filterRecordsByRootLineBlacklist(array &$records) { $records = array_filter( $records, @@ -142,10 +146,21 @@ class TcaTableService * Adjust record accordingly to configuration. * @param array &$record */ - public function prepareRecord(array &$record) : void + public function prepareRecord(array &$record) { $this->relationResolver->resolveRelationsForRecord($this, $record); + try { + foreach ($this->configuration->get('indexing.' . $this->tableName . '.dataProcessing') as $configuration) { + $dataProcessor = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance($configuration['_typoScriptNodeValue']); + if ($dataProcessor instanceof ProcessorInterface) { + $record = $dataProcessor->processRecord($record, $configuration); + } + } + } catch (InvalidConfigurationArgumentException $e) { + // Nothing to do. + } + if (isset($record['uid']) && !isset($record['search_identifier'])) { $record['search_identifier'] = $record['uid']; } @@ -154,10 +169,22 @@ class TcaTableService } } - public function getWhereClause() : Where + /** + * @return string + */ + public function getWhereClause() { - $parameters = []; - $whereClause = $this->getSystemWhereClause(); + $whereClause = '1=1' + . BackendUtility::BEenableFields($this->tableName) + . BackendUtility::deleteClause($this->tableName) + . ' AND pages.no_search = 0' + ; + + if ($this->tableName !== 'pages') { + $whereClause .= BackendUtility::BEenableFields('pages') + . BackendUtility::deleteClause('pages') + ; + } $userDefinedWhere = $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.additionalWhereClause'); if (is_string($userDefinedWhere)) { @@ -165,16 +192,22 @@ class TcaTableService } if ($this->isBlacklistedRootLineConfigured()) { - $parameters[':blacklistedRootLine'] = $this->getBlacklistedRootLine(); - $whereClause .= ' AND pages.uid NOT IN (:blacklistedRootLine)' - . ' AND pages.pid NOT IN (:blacklistedRootLine)'; + $whereClause .= ' AND pages.uid NOT IN (' + . implode(',', $this->getBlacklistedRootLine()) + . ')' + . ' AND pages.pid NOT IN (' + . implode(',', $this->getBlacklistedRootLine()) + . ')'; } $this->logger->debug('Generated where clause.', [$this->tableName, $whereClause]); - return new Where($whereClause, $parameters); + return $whereClause; } - public function getFields() : array + /** + * @return string + */ + public function getFields() { $fields = array_merge( ['uid','pid'], @@ -191,46 +224,14 @@ class TcaTableService } $this->logger->debug('Generated fields.', [$this->tableName, $fields]); - return $fields; - } - - public function getJoins() : array - { - if ($this->tableName === 'pages') { - return []; - } - - return [ - new Join('pages', 'pages.uid = ' . $this->tableName . '.pid'), - ]; - } - - /** - * Generate SQL for TYPO3 as a system, to make sure only available records - * are fetched. - */ - public function getSystemWhereClause() : string - { - $whereClause = '1=1' - . BackendUtility::BEenableFields($this->tableName) - . BackendUtility::deleteClause($this->tableName) - . ' AND pages.no_search = 0' - ; - - if ($this->tableName !== 'pages') { - $whereClause .= BackendUtility::BEenableFields('pages') - . BackendUtility::deleteClause('pages') - ; - } - - return $whereClause; + return implode(',', $fields); } /** * @param string * @return bool */ - protected function isSystemField($columnName) : bool + protected function isSystemField($columnName) { $systemFields = [ // Versioning fields, @@ -254,7 +255,7 @@ class TcaTableService * @return array * @throws InvalidArgumentException */ - public function getColumnConfig($columnName) : array + public function getColumnConfig($columnName) { if (!isset($this->tca['columns'][$columnName])) { throw new InvalidArgumentException( @@ -277,7 +278,7 @@ class TcaTableService * @param array &$record * @return bool */ - protected function isRecordBlacklistedByRootline(array &$record) : bool + protected function isRecordBlacklistedByRootline(array &$record) { $pageUid = $record['pid']; if ($this->tableName === 'pages') { @@ -334,7 +335,7 @@ class TcaTableService * * @return bool */ - protected function isBlackListedRootLineConfigured() : bool + protected function isBlackListedRootLineConfigured() { return (bool) $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist'); } @@ -344,7 +345,7 @@ class TcaTableService * * @return array */ - protected function getBlackListedRootLine() : array + protected function getBlackListedRootLine() { return GeneralUtility::intExplode(',', $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist')); } diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 9455f80..c4b66c9 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -22,9 +22,9 @@ namespace Codappix\SearchCore\Domain\Search; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\InvalidArgumentException; -use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Connection\Elasticsearch\Query; use Codappix\SearchCore\Connection\SearchRequestInterface; +use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Extbase\Utility\ArrayUtility; class QueryFactory @@ -109,20 +109,18 @@ class QueryFactory return; } - $query = ArrayUtility::setValueByPath( - $query, - 'query.bool.must.0.match._all.query', - $searchRequest->getSearchTerm() - ); + $matchExpression = [ + 'type' => 'most_fields', + 'query' => $searchRequest->getSearchTerm(), + 'fields' => GeneralUtility::trimExplode(',', $this->configuration->get('searching.fields')), + ]; $minimumShouldMatch = $this->configuration->getIfExists('searching.minimumShouldMatch'); if ($minimumShouldMatch) { - $query = ArrayUtility::setValueByPath( - $query, - 'query.bool.must.0.match._all.minimum_should_match', - $minimumShouldMatch - ); + $matchExpression['minimum_should_match'] = $minimumShouldMatch; } + + $query = ArrayUtility::setValueByPath($query, 'query.bool.must.0.multi_match', $matchExpression); } /** @@ -150,13 +148,15 @@ class QueryFactory ]; } - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ - 'query' => [ - 'bool' => [ - 'should' => $boostQueryParts, + if (!empty($boostQueryParts)) { + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ + 'query' => [ + 'bool' => [ + 'should' => $boostQueryParts, + ], ], - ], - ]); + ]); + } } /** diff --git a/Classes/Domain/Service/DataHandler.php b/Classes/Domain/Service/DataHandler.php index 6ac8069..4b0386e 100644 --- a/Classes/Domain/Service/DataHandler.php +++ b/Classes/Domain/Service/DataHandler.php @@ -23,7 +23,6 @@ namespace Codappix\SearchCore\Domain\Service; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Domain\Index\IndexerFactory; use Codappix\SearchCore\Domain\Index\NoMatchingIndexerException; -use Codappix\SearchCore\Domain\Index\TcaIndexer; use TYPO3\CMS\Core\SingletonInterface as Singleton; use TYPO3\CMS\Core\Utility\GeneralUtility; diff --git a/Classes/Hook/DataHandler.php b/Classes/Hook/DataHandler.php index d0eb1ba..3c4902a 100644 --- a/Classes/Hook/DataHandler.php +++ b/Classes/Hook/DataHandler.php @@ -21,7 +21,6 @@ namespace Codappix\SearchCore\Hook; */ use Codappix\SearchCore\Configuration\NoConfigurationException; -use Codappix\SearchCore\Domain\Index\NoMatchingIndexerException; use Codappix\SearchCore\Domain\Service\DataHandler as OwnDataHandler; use TYPO3\CMS\Backend\Utility\BackendUtility; use TYPO3\CMS\Core\DataHandling\DataHandler as CoreDataHandler; diff --git a/Documentation/source/readme.rst b/Documentation/source/readme.rst index ba77ae8..cc516d1 100644 --- a/Documentation/source/readme.rst +++ b/Documentation/source/readme.rst @@ -28,4 +28,3 @@ We are also focusing on Code Quality and Testing through `travis ci`_, `scrutini .. _travis ci: https://travis-ci.org/Codappix/search_core .. _scrutinizer: https://scrutinizer-ci.com/g/Codappix/search_core/inspections .. _codacy: https://www.codacy.com/app/Codappix/search_core/dashboard - diff --git a/Makefile b/Makefile index b6bc72b..72a1b9b 100644 --- a/Makefile +++ b/Makefile @@ -4,15 +4,15 @@ current_dir := $(dir $(mkfile_path)) TYPO3_WEB_DIR := $(current_dir).Build/web TYPO3_PATH_ROOT := $(current_dir).Build/web # Allow different versions on travis -TYPO3_VERSION ?= ~8.7 -typo3DatabaseName ?= "searchcore_test2" +TYPO3_VERSION ?= ~7.6 +typo3DatabaseName ?= "searchcore_test" typo3DatabaseUsername ?= "dev" typo3DatabasePassword ?= "dev" typo3DatabaseHost ?= "127.0.0.1" .PHONY: install install: clean - COMPOSER_PROCESS_TIMEOUT=1000 composer require -vv --dev --prefer-dist --ignore-platform-reqs typo3/cms="$(TYPO3_VERSION)" + COMPOSER_PROCESS_TIMEOUT=1000 composer require -vv --dev --prefer-source typo3/cms="$(TYPO3_VERSION)" git checkout composer.json functionalTests: diff --git a/Tests/Functional/Connection/Elasticsearch/FilterTest.php b/Tests/Functional/Connection/Elasticsearch/FilterTest.php index e364a4a..2430833 100644 --- a/Tests/Functional/Connection/Elasticsearch/FilterTest.php +++ b/Tests/Functional/Connection/Elasticsearch/FilterTest.php @@ -48,14 +48,14 @@ class FilterTest extends AbstractFunctionalTestCase $searchService = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) ->get(SearchService::class); - $searchRequest = new SearchRequest('Search Word'); + $searchRequest = new SearchRequest(); $result = $searchService->search($searchRequest); $this->assertSame(2, count($result), 'Did not receive both indexed elements without filter.'); $searchRequest->setFilter(['CType' => 'HTML']); $result = $searchService->search($searchRequest); - $this->assertSame(5, $result->getResults()[0]['uid'], 'Did not get the expected result entry.'); + $this->assertSame(5, (int) $result->getResults()[0]['uid'], 'Did not get the expected result entry.'); $this->assertSame(1, count($result), 'Did not receive the single filtered element.'); } @@ -73,7 +73,7 @@ class FilterTest extends AbstractFunctionalTestCase $searchService = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) ->get(SearchService::class); - $searchRequest = new SearchRequest('Search Word'); + $searchRequest = new SearchRequest(); $result = $searchService->search($searchRequest); $this->assertSame(1, count($result->getFacets()), 'Did not receive the single defined facet.'); diff --git a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php index 022fb3b..02e6ca3 100644 --- a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php +++ b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php @@ -175,7 +175,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase ['_source' => [ 'uid' => '11', 'CType' => 'Header', // Testing items - 'categories' => ['Category 2', 'Category 1'], // Testing mm + 'categories' => ['Category 2', 'Category 1'], // Testing mm (with sorting) ]], $response->getData()['hits']['hits'][0], false, diff --git a/Tests/Functional/Fixtures/BasicSetup.ts b/Tests/Functional/Fixtures/BasicSetup.ts index 1e2b3a9..9795b0f 100644 --- a/Tests/Functional/Fixtures/BasicSetup.ts +++ b/Tests/Functional/Fixtures/BasicSetup.ts @@ -37,6 +37,7 @@ plugin { } searching { + fields = search_title facets { contentTypes { field = CType diff --git a/Tests/Functional/Fixtures/Indexing/IndexTcaTable.xml b/Tests/Functional/Fixtures/Indexing/IndexTcaTable.xml index c236f07..3692135 100644 --- a/Tests/Functional/Fixtures/Indexing/IndexTcaTable.xml +++ b/Tests/Functional/Fixtures/Indexing/IndexTcaTable.xml @@ -29,7 +29,7 @@ 72 header
endtime hidden record
- + Some content 0 0 0 @@ -49,7 +49,7 @@ 72 header
Hidden record
- + Some content 0 0 0 diff --git a/Tests/Functional/Fixtures/Indexing/UserWhereClause.xml b/Tests/Functional/Fixtures/Indexing/UserWhereClause.xml index 12347ef..b9d9c7c 100644 --- a/Tests/Functional/Fixtures/Indexing/UserWhereClause.xml +++ b/Tests/Functional/Fixtures/Indexing/UserWhereClause.xml @@ -9,7 +9,7 @@ 72 header
Also indexable record
- + Some content 0 0 0 @@ -29,7 +29,7 @@ 72 div
Not indexed by user where ctype condition
- + Some content 0 0 0 diff --git a/Tests/Functional/FunctionalTests.xml b/Tests/Functional/FunctionalTests.xml index a318f65..d42ef21 100644 --- a/Tests/Functional/FunctionalTests.xml +++ b/Tests/Functional/FunctionalTests.xml @@ -1,7 +1,7 @@ with( $this->equalTo('tt_content'), $this->callback(function ($record) { - return isset($record['uid']) && $record['uid'] === 1 - && isset($record['pid']) && $record['pid'] === 1 - && isset($record['colPos']) && $record['colPos'] === 1 + return isset($record['uid']) && $record['uid'] == 1 + && isset($record['pid']) && $record['pid'] == 1 + && isset($record['colPos']) && $record['colPos'] == 1 ; }) ); @@ -100,9 +100,9 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest ->with( $this->equalTo('tt_content'), $this->callback(function ($record) { - return isset($record['uid']) && $record['uid'] === '2' - && isset($record['pid']) && $record['pid'] === 1 - && isset($record['header']) && $record['header'] === 'a new record' + return isset($record['uid']) && $record['uid'] == 2 + && isset($record['pid']) && $record['pid'] == 1 + && isset($record['header']) && $record['header'] == 'a new record' ; }) ); diff --git a/Tests/Unit/Connection/Elasticsearch/IndexFactoryTest.php b/Tests/Unit/Connection/Elasticsearch/IndexFactoryTest.php index c73fe36..4ede3cb 100644 --- a/Tests/Unit/Connection/Elasticsearch/IndexFactoryTest.php +++ b/Tests/Unit/Connection/Elasticsearch/IndexFactoryTest.php @@ -38,6 +38,7 @@ class IndexFactoryTest extends AbstractUnitTestCase $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock(); $this->subject = new IndexFactory($this->configuration); + $this->subject->injectLogger($this->getMockedLogger()); } /** diff --git a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php index 7fb0280..9e88d4b 100644 --- a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php +++ b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php @@ -60,18 +60,10 @@ class TcaTableServiceTest extends AbstractUnitTestCase ->method('getIfExists') ->withConsecutive(['indexing.table.additionalWhereClause'], ['indexing.table.rootLineBlacklist']) ->will($this->onConsecutiveCalls(null, false)); - $this->subject->expects($this->once()) - ->method('getSystemWhereClause') - ->will($this->returnValue('1=1 AND pages.no_search = 0')); - $whereClause = $this->subject->getWhereClause(); $this->assertSame( '1=1 AND pages.no_search = 0', - $whereClause->getStatement() - ); - $this->assertSame( - [], - $whereClause->getParameters() + $this->subject->getWhereClause() ); } @@ -84,18 +76,10 @@ class TcaTableServiceTest extends AbstractUnitTestCase ->method('getIfExists') ->withConsecutive(['indexing.table.additionalWhereClause'], ['indexing.table.rootLineBlacklist']) ->will($this->onConsecutiveCalls('table.field = "someValue"', false)); - $this->subject->expects($this->once()) - ->method('getSystemWhereClause') - ->will($this->returnValue('1=1 AND pages.no_search = 0')); - $whereClause = $this->subject->getWhereClause(); $this->assertSame( '1=1 AND pages.no_search = 0 AND table.field = "someValue"', - $whereClause->getStatement() - ); - $this->assertSame( - [], - $whereClause->getParameters() + $this->subject->getWhereClause() ); } } diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index 54fdc2a..f719fdf 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -50,13 +50,11 @@ class QueryFactoryTest extends AbstractUnitTestCase /** * @test */ - public function creatonOfQueryWorksInGeneral() + public function creationOfQueryWorksInGeneral() { - $searchRequest = new SearchRequest('SearchWord'); + $this->mockConfiguration(); - $this->configuration->expects($this->any()) - ->method('get') - ->will($this->throwException(new InvalidArgumentException)); + $searchRequest = new SearchRequest('SearchWord'); $query = $this->subject->create($searchRequest); $this->assertInstanceOf( @@ -71,9 +69,7 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function filterIsAddedToQuery() { - $this->configuration->expects($this->any()) - ->method('get') - ->will($this->throwException(new InvalidArgumentException)); + $this->mockConfiguration(); $searchRequest = new SearchRequest('SearchWord'); $searchRequest->setFilter(['field' => 'content']); @@ -83,7 +79,7 @@ class QueryFactoryTest extends AbstractUnitTestCase [ ['term' => ['field' => 'content']] ], - $query->toArray()['query']['bool']['filter'], + $query->toArray()['query']['function_score']['query']['bool']['filter'], 'Filter was not added to query.' ); } @@ -93,9 +89,7 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function emptyFilterIsNotAddedToQuery() { - $this->configuration->expects($this->any()) - ->method('get') - ->will($this->throwException(new InvalidArgumentException)); + $this->mockConfiguration(); $searchRequest = new SearchRequest('SearchWord'); $searchRequest->setFilter([ @@ -112,7 +106,7 @@ class QueryFactoryTest extends AbstractUnitTestCase $query = $this->subject->create($searchRequest); $this->assertSame( null, - $query->toArray()['query']['bool']['filter'], + $query->toArray()['query']['function_score']['query']['bool']['filter'], 'Filter was added to query, even if no filter exists.' ); } @@ -122,9 +116,8 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function facetsAreAddedToQuery() { - $this->configuration->expects($this->any()) - ->method('get') - ->will($this->throwException(new InvalidArgumentException)); + $this->mockConfiguration(); + $searchRequest = new SearchRequest('SearchWord'); $searchRequest->addFacet(new FacetRequest('Identifier', 'FieldName')); $searchRequest->addFacet(new FacetRequest('Identifier 2', 'FieldName 2')); @@ -153,9 +146,8 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function sizeIsAddedToQuery() { - $this->configuration->expects($this->any()) - ->method('get') - ->will($this->throwException(new InvalidArgumentException)); + $this->mockConfiguration(); + $searchRequest = new SearchRequest('SearchWord'); $searchRequest->setLimit(45); $searchRequest->setOffset(35); @@ -178,10 +170,9 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function searchTermIsAddedToQuery() { + $this->mockConfiguration(); + $searchRequest = new SearchRequest('SearchWord'); - $this->configuration->expects($this->any()) - ->method('get') - ->will($this->throwException(new InvalidArgumentException)); $query = $this->subject->create($searchRequest); $this->assertSame( @@ -189,16 +180,18 @@ class QueryFactoryTest extends AbstractUnitTestCase 'bool' => [ 'must' => [ [ - 'match' => [ - '_all' => [ - 'query' => 'SearchWord', + 'multi_match' => [ + 'type' => 'most_fields', + 'query' => 'SearchWord', + 'fields' => [ + 0 => 'test_field', ], ], ], ], ], ], - $query->toArray()['query'], + $query->toArray()['query']['function_score']['query'], 'Search term was not added to query as expected.' ); } @@ -208,14 +201,13 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function minimumShouldMatchIsAddedToQuery() { - $searchRequest = new SearchRequest('SearchWord'); $this->configuration->expects($this->once()) ->method('getIfExists') ->with('searching.minimumShouldMatch') ->willReturn('50%'); - $this->configuration->expects($this->any()) - ->method('get') - ->will($this->throwException(new InvalidArgumentException)); + $this->mockConfiguration(); + + $searchRequest = new SearchRequest('SearchWord'); $query = $this->subject->create($searchRequest); $this->assertArraySubset( @@ -223,16 +215,14 @@ class QueryFactoryTest extends AbstractUnitTestCase 'bool' => [ 'must' => [ [ - 'match' => [ - '_all' => [ - 'minimum_should_match' => '50%', - ], + 'multi_match' => [ + 'minimum_should_match' => '50%', ], ], ], ], ], - $query->toArray()['query'], + $query->toArray()['query']['function_score']['query'], 'minimum_should_match was not added to query as configured.' ); } @@ -244,10 +234,11 @@ class QueryFactoryTest extends AbstractUnitTestCase { $searchRequest = new SearchRequest('SearchWord'); - $this->configuration->expects($this->exactly(2)) + $this->configuration->expects($this->exactly(3)) ->method('get') - ->withConsecutive(['searching.boost'], ['searching.fieldValueFactor']) + ->withConsecutive(['searching.fields'], ['searching.boost'], ['searching.fieldValueFactor']) ->will($this->onConsecutiveCalls( + 'test_field', [ 'search_title' => 3, 'search_abstract' => 1.5, @@ -292,10 +283,11 @@ class QueryFactoryTest extends AbstractUnitTestCase 'factor' => '2', 'missing' => '1', ]; - $this->configuration->expects($this->exactly(2)) + $this->configuration->expects($this->exactly(3)) ->method('get') - ->withConsecutive(['searching.boost'], ['searching.fieldValueFactor']) + ->withConsecutive(['searching.fields'], ['searching.boost'], ['searching.fieldValueFactor']) ->will($this->onConsecutiveCalls( + 'test_field', $this->throwException(new InvalidArgumentException), $fieldConfig )); @@ -308,9 +300,11 @@ class QueryFactoryTest extends AbstractUnitTestCase 'bool' => [ 'must' => [ [ - 'match' => [ - '_all' => [ - 'query' => 'SearchWord', + 'multi_match' => [ + 'type' => 'most_fields', + 'query' => 'SearchWord', + 'fields' => [ + 0 => 'test_field', ], ], ], @@ -330,17 +324,27 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function emptySearchStringWillNotAddSearchToQuery() { + $this->mockConfiguration(); + $searchRequest = new SearchRequest(); - $this->configuration->expects($this->any()) - ->method('get') - ->will($this->throwException(new InvalidArgumentException)); - $query = $this->subject->create($searchRequest); - $this->assertInstanceOf( - stdClass, - $query->toArray()['query']['match_all'], + $this->assertNull( + $query->toArray()['query']['function_score']['query'], 'Empty search request does not create expected query.' ); } + + protected function mockConfiguration() + { + $this->configuration->expects($this->any()) + ->method('get') + ->will($this->returnCallback(function ($option) { + if ($option === 'searching.fields') { + return 'test_field'; + } + + return $this->throwException(new InvalidArgumentException); + })); + } } diff --git a/Tests/Unit/UnitTests.xml b/Tests/Unit/UnitTests.xml index 6486594..6456405 100644 --- a/Tests/Unit/UnitTests.xml +++ b/Tests/Unit/UnitTests.xml @@ -1,7 +1,7 @@ =7.1.0", - "typo3/cms": "~8.7", + "php": ">=5.6.0", + "typo3/cms": "~7.6", "ruflin/elastica": "~3.2" }, "require-dev": { - "typo3/testing-framework": "~1.1.0", - "phpunit/phpunit": "~6.2.0" + "phpunit/phpunit": "~5.7.0" }, "config": { "optimize-autoloader": true, diff --git a/ext_emconf.php b/ext_emconf.php index 4d342a5..118e2df 100644 --- a/ext_emconf.php +++ b/ext_emconf.php @@ -7,8 +7,8 @@ $EM_CONF[$_EXTKEY] = [ 'clearCacheOnLoad' => 1, 'constraints' => [ 'depends' => [ - 'typo3' => '8.7.0-8.7.99', - 'php' => '7.1.0-7.99.99' + 'typo3' => '7.6.0-7.6.99', + 'php' => '5.6.0-7.99.99' ], 'conflicts' => [], ], diff --git a/ext_localconf.php b/ext_localconf.php index 54bf43b..882f6ad 100644 --- a/ext_localconf.php +++ b/ext_localconf.php @@ -11,7 +11,7 @@ call_user_func( 'SC_OPTIONS' => [ 'extbase' => [ 'commandControllers' => [ - Codappix\SearchCore\Command\IndexCommandController::class, + $extensionKey . '::index' => Codappix\SearchCore\Command\IndexCommandController::class, ], ], 't3lib/class.t3lib_tcemain.php' => [