Merge branch 'develop' into feature/75-make-index-configurable

# Conflicts:
#	Documentation/source/configuration/connections.rst
#	Documentation/source/connections.rst
This commit is contained in:
Justus Moroni 2018-05-17 23:51:55 +02:00
commit 2b56d4c2d5
129 changed files with 3906 additions and 1175 deletions

96
.phan/config.php Normal file
View file

@ -0,0 +1,96 @@
<?php
/**
* This configuration will be read and overlaid on top of the
* default configuration. Command line arguments will be applied
* after this file is read.
*/
return [
// Supported values: '7.0', '7.1', '7.2', null.
// If this is set to null,
// then Phan assumes the PHP version which is closest to the minor version
// of the php executable used to execute phan.
"target_php_version" => '7.0',
// Override to hardcode existence and types of (non-builtin) globals.
// Class names should be prefixed with '\\'.
// (E.g. ['_FOO' => '\\FooClass', 'page' => '\\PageClass', 'userId' => 'int'])
'globals_type_map' => [
'_EXTKEY' => 'string',
'EM_CONF' => 'array',
],
// A list of directories that should be parsed for class and
// method information. After excluding the directories
// defined in exclude_analysis_directory_list, the remaining
// files will be statically analyzed for errors.
//
// Thus, both first-party and third-party code being used by
// your application should be included in this list.
'directory_list' => [
'Classes',
'.Build/vendor',
],
// A list of files to include in analysis
'file_list' => [
'ext_emconf.php',
'ext_tables.php',
'ext_localconf.php',
],
// A directory list that defines files that will be excluded
// from static analysis, but whose class and method
// information should be included.
//
// Generally, you'll want to include the directories for
// third-party code (such as "vendor/") in this list.
//
// n.b.: If you'd like to parse but not analyze 3rd
// party code, directories containing that code
// should be added to the `directory_list` as
// to `exclude_analysis_directory_list`.
"exclude_analysis_directory_list" => [
'.Build/vendor'
],
// A list of directories that should be parsed for class and
// method information. After excluding the directories
// defined in exclude_analysis_directory_list, the remaining
// files will be statically analyzed for errors.
//
// Thus, both first-party and third-party code being used by
// your application should be included in this list.
'directory_list' => [
'Classes',
// 'Tests',
'.Build/vendor',
],
// The number of processes to fork off during the analysis phase.
'processes' => 3,
// Add any issue types (such as 'PhanUndeclaredMethod')
// here to inhibit them from being reported
'suppress_issue_types' => [
'PhanDeprecatedFunction', // For now
'PhanParamTooMany', // For now, due to ObjectManager->get()
],
// A list of plugin files to execute.
// See https://github.com/phan/phan/tree/master/.phan/plugins for even more.
// (Pass these in as relative paths.
// The 0.10.2 release will allow passing 'AlwaysReturnPlugin' if referring to a plugin that is bundled with Phan)
'plugins' => [
// checks if a function, closure or method unconditionally returns.
'AlwaysReturnPlugin', // can also be written as 'vendor/phan/phan/.phan/plugins/AlwaysReturnPlugin.php'
// Checks for syntactically unreachable statements in
// the global scope or function bodies.
'UnreachableCodePlugin',
'DollarDollarPlugin',
'DuplicateArrayKeyPlugin',
'PregRegexCheckerPlugin',
'PrintfCheckerPlugin',
],
];

View file

@ -1,3 +1,11 @@
build:
nodes:
analysis:
project_setup:
override: true
tests:
override: [php-scrutinizer-run]
filter: filter:
excluded_paths: excluded_paths:
- 'Configuration/*' - 'Configuration/*'
@ -19,7 +27,7 @@ tools:
php_hhvm: php_hhvm:
enabled: true enabled: true
config: config:
use_undeclared_constant: false use_undeclared_constant: false
php_mess_detector: php_mess_detector:
enabled: true enabled: true
@ -34,5 +42,5 @@ tools:
enabled: true enabled: true
# We generate code coverage during tests at travis and will send them here # We generate code coverage during tests at travis and will send them here
external_code_coverage: external_code_coverage:
runs: 2 runs: 2
timeout: 1200 timeout: 1200

View file

@ -5,13 +5,15 @@ addons:
packages: packages:
- oracle-java8-set-default - oracle-java8-set-default
before_install: before_install:
- curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.2.0.deb && sudo dpkg -i --force-confnew elasticsearch-5.2.0.deb && sudo service elasticsearch start - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.2.0.deb && sudo dpkg -i --force-confnew elasticsearch-5.2.0.deb && sudo service elasticsearch restart
- mysql -u root -e 'GRANT ALL ON `typo3_ci_ft%`.* TO travis@127.0.0.1;' - mysql -u root -e 'GRANT ALL ON `typo3_ci_ft%`.* TO travis@127.0.0.1;'
language: php language: php
php: php:
- 7.0
- 7.1 - 7.1
- 7.2
env: env:
global: global:
@ -23,6 +25,9 @@ env:
- typo3DatabaseHost="127.0.0.1" - typo3DatabaseHost="127.0.0.1"
- typo3DatabaseUsername="travis" - typo3DatabaseUsername="travis"
- typo3DatabasePassword="" - typo3DatabasePassword=""
matrix:
- TYPO3_VERSION="~7.6"
- TYPO3_VERSION="~8.7"
matrix: matrix:
fast_finish: true fast_finish: true
@ -33,6 +38,7 @@ services:
install: make install install: make install
script: script:
- make cgl
- make unitTests - make unitTests
- make functionalTests - make functionalTests

View file

@ -22,7 +22,6 @@ namespace Codappix\SearchCore\Command;
use Codappix\SearchCore\Domain\Index\IndexerFactory; use Codappix\SearchCore\Domain\Index\IndexerFactory;
use Codappix\SearchCore\Domain\Index\NoMatchingIndexerException; use Codappix\SearchCore\Domain\Index\NoMatchingIndexerException;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Extbase\Mvc\Controller\CommandController; use TYPO3\CMS\Extbase\Mvc\Controller\CommandController;
/** /**
@ -48,7 +47,7 @@ class IndexCommandController extends CommandController
* *
* @param string $identifier * @param string $identifier
*/ */
public function indexCommand($identifier) public function indexCommand(string $identifier)
{ {
try { try {
$this->indexerFactory->getIndexer($identifier)->indexAllDocuments(); $this->indexerFactory->getIndexer($identifier)->indexAllDocuments();
@ -63,7 +62,7 @@ class IndexCommandController extends CommandController
* *
* @param string $identifier * @param string $identifier
*/ */
public function deleteCommand($identifier) public function deleteCommand(string $identifier)
{ {
try { try {
$this->indexerFactory->getIndexer($identifier)->delete(); $this->indexerFactory->getIndexer($identifier)->delete();

View file

@ -0,0 +1,56 @@
<?php
namespace Codappix\SearchCore\Compatibility;
/*
* Copyright (C) 2018 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Core\Utility\VersionNumberUtility;
use TYPO3\CMS\Extbase\Object\Container\Container;
/**
* Register different concrete implementations, depending on current TYPO3 version.
* This way we can provide working implementations for multiple TYPO3 versions.
*/
class ImplementationRegistrationService
{
public static function registerImplementations()
{
$container = GeneralUtility::makeInstance(Container::class);
if (VersionNumberUtility::convertVersionNumberToInteger(TYPO3_version) >= 8000000) {
$container->registerImplementation(
\Codappix\SearchCore\Compatibility\TypoScriptServiceInterface::class,
\Codappix\SearchCore\Compatibility\TypoScriptService::class
);
$container->registerImplementation(
\Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface::class,
\Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService::class
);
} else {
$container->registerImplementation(
\Codappix\SearchCore\Compatibility\TypoScriptServiceInterface::class,
\Codappix\SearchCore\Compatibility\TypoScriptService76::class
);
$container->registerImplementation(
\Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface::class,
\Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService76::class
);
}
}
}

View file

@ -0,0 +1,31 @@
<?php
namespace Codappix\SearchCore\Compatibility;
/*
* Copyright (C) 2018 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
use TYPO3\CMS\Core\TypoScript\TypoScriptService as CoreTypoScriptService;
/**
* Used since TYPO3 CMS 8.7.
*/
class TypoScriptService extends CoreTypoScriptService implements TypoScriptServiceInterface
{
}

View file

@ -0,0 +1,31 @@
<?php
namespace Codappix\SearchCore\Compatibility;
/*
* Copyright (C) 2018 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
use TYPO3\CMS\Extbase\Service\TypoScriptService as CoreTypoScriptService;
/**
* Used before TYPO3 CMS 8.7.
*/
class TypoScriptService76 extends CoreTypoScriptService implements TypoScriptServiceInterface
{
}

View file

@ -0,0 +1,30 @@
<?php
namespace Codappix\SearchCore\Compatibility;
/*
* Copyright (C) 2018 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
/**
* Allows to use DI configuration to switch concrete implementation, depending
* on current TYPO3 Version.
*/
interface TypoScriptServiceInterface
{
public function convertPlainArrayToTypoScriptArray(array $plainArray);
}

View file

@ -39,7 +39,6 @@ class ConfigurationContainer implements ConfigurationContainerInterface
/** /**
* Inject settings via ConfigurationManager. * Inject settings via ConfigurationManager.
* *
* @param ConfigurationManagerInterface $configurationManager
* @throws NoConfigurationException * @throws NoConfigurationException
*/ */
public function injectConfigurationManager(ConfigurationManagerInterface $configurationManager) public function injectConfigurationManager(ConfigurationManagerInterface $configurationManager)
@ -59,7 +58,7 @@ class ConfigurationContainer implements ConfigurationContainerInterface
* @return mixed * @return mixed
* @throws InvalidArgumentException * @throws InvalidArgumentException
*/ */
public function get($path) public function get(string $path)
{ {
$value = ArrayUtility::getValueByPath($this->settings, $path); $value = ArrayUtility::getValueByPath($this->settings, $path);
@ -77,7 +76,7 @@ class ConfigurationContainer implements ConfigurationContainerInterface
* @param string $path In dot notation. * @param string $path In dot notation.
* @return mixed * @return mixed
*/ */
public function getIfExists($path) public function getIfExists(string $path)
{ {
return ArrayUtility::getValueByPath($this->settings, $path); return ArrayUtility::getValueByPath($this->settings, $path);
} }

View file

@ -20,13 +20,11 @@ namespace Codappix\SearchCore\Configuration;
* 02110-1301, USA. * 02110-1301, USA.
*/ */
use TYPO3\CMS\Core\SingletonInterface as Singleton;
/** /**
* Container of all configurations for extension. * Container of all configurations for extension.
* Always inject this to have a single place for configuration and parsing only once. * Always inject this to have a single place for configuration.
*/ */
interface ConfigurationContainerInterface extends Singleton interface ConfigurationContainerInterface
{ {
/** /**
* Returns the option defined by section and key. * Returns the option defined by section and key.
@ -37,7 +35,7 @@ interface ConfigurationContainerInterface extends Singleton
* *
* @throws InvalidArgumentException * @throws InvalidArgumentException
*/ */
public function get($path); public function get(string $path);
/** /**
* Same as get but will not throw an exception but return null. * Same as get but will not throw an exception but return null.
@ -45,5 +43,5 @@ interface ConfigurationContainerInterface extends Singleton
* @param string $path In dot notation. * @param string $path In dot notation.
* @return mixed|null * @return mixed|null
*/ */
public function getIfExists($path); public function getIfExists(string $path);
} }

View file

@ -28,62 +28,44 @@ interface ConnectionInterface
/** /**
* Will add a new document. * Will add a new document.
* *
* @param string $documentType
* @param array $document
*
* @return void * @return void
*/ */
public function addDocument($documentType, array $document); public function addDocument(string $documentType, array $document);
/** /**
* Add the given documents. * Add the given documents.
* *
* @param string $documentType
* @param array $documents
*
* @return void * @return void
*/ */
public function addDocuments($documentType, array $documents); public function addDocuments(string $documentType, array $documents);
/** /**
* Will update an existing document. * Will update an existing document.
* *
* NOTE: Batch updating is not yet supported. * NOTE: Batch updating is not yet supported.
* *
* @param string $documentType
* @param array $document
*
* @return void * @return void
*/ */
public function updateDocument($documentType, array $document); public function updateDocument(string $documentType, array $document);
/** /**
* Will remove an existing document. * Will remove an existing document.
* *
* NOTE: Batch deleting is not yet supported. * NOTE: Batch deleting is not yet supported.
* *
* @param string $documentType
* @param int $identifier
*
* @return void * @return void
*/ */
public function deleteDocument($documentType, $identifier); public function deleteDocument(string $documentType, string $identifier);
/** /**
* Search by given request and return result. * Search by given request and return result.
*
* @param SearchRequestInterface $searchRequest
*
* @return SearchResultInterface
*/ */
public function search(SearchRequestInterface $searchRequest); public function search(SearchRequestInterface $searchRequest) : SearchResultInterface;
/** /**
* Will delete the whole index / db. * Will delete the whole index / db.
* *
* @param string $documentType
*
* @return void * @return void
*/ */
public function deleteIndex($documentType); public function deleteIndex(string $documentType);
} }

View file

@ -112,7 +112,7 @@ class Elasticsearch implements Singleton, ConnectionInterface
$this->queryFactory = $queryFactory; $this->queryFactory = $queryFactory;
} }
public function addDocument($documentType, array $document) public function addDocument(string $documentType, array $document)
{ {
$this->withType( $this->withType(
$documentType, $documentType,
@ -122,7 +122,7 @@ class Elasticsearch implements Singleton, ConnectionInterface
); );
} }
public function deleteDocument($documentType, $identifier) public function deleteDocument(string $documentType, string $identifier)
{ {
try { try {
$this->withType( $this->withType(
@ -132,11 +132,14 @@ class Elasticsearch implements Singleton, ConnectionInterface
} }
); );
} catch (\Elastica\Exception\NotFoundException $exception) { } catch (\Elastica\Exception\NotFoundException $exception) {
$this->logger->debug('Tried to delete document in index, which does not exist.', [$documentType, $identifier]); $this->logger->debug(
'Tried to delete document in index, which does not exist.',
[$documentType, $identifier]
);
} }
} }
public function updateDocument($documentType, array $document) public function updateDocument(string $documentType, array $document)
{ {
$this->withType( $this->withType(
$documentType, $documentType,
@ -146,7 +149,7 @@ class Elasticsearch implements Singleton, ConnectionInterface
); );
} }
public function addDocuments($documentType, array $documents) public function addDocuments(string $documentType, array $documents)
{ {
$this->withType( $this->withType(
$documentType, $documentType,
@ -156,7 +159,7 @@ class Elasticsearch implements Singleton, ConnectionInterface
); );
} }
public function deleteIndex($documentType) public function deleteIndex(string $documentType)
{ {
$index = $this->connection->getClient()->getIndex($this->indexFactory->getIndexName()); $index = $this->connection->getClient()->getIndex($this->indexFactory->getIndexName());
@ -173,11 +176,8 @@ class Elasticsearch implements Singleton, ConnectionInterface
/** /**
* Execute given callback with Elastica Type based on provided documentType * Execute given callback with Elastica Type based on provided documentType
*
* @param string $documentType
* @param callable $callback
*/ */
protected function withType($documentType, callable $callback) protected function withType(string $documentType, callable $callback)
{ {
$type = $this->getType($documentType); $type = $this->getType($documentType);
// TODO: Check whether it's to heavy to send it so often e.g. for every single document. // TODO: Check whether it's to heavy to send it so often e.g. for every single document.
@ -191,12 +191,7 @@ class Elasticsearch implements Singleton, ConnectionInterface
$type->getIndex()->refresh(); $type->getIndex()->refresh();
} }
/** public function search(SearchRequestInterface $searchRequest) : SearchResultInterface
* @param SearchRequestInterface $searchRequest
*
* @return SearchResultInterface
*/
public function search(SearchRequestInterface $searchRequest)
{ {
$this->logger->debug('Search for', [$searchRequest->getSearchTerm()]); $this->logger->debug('Search for', [$searchRequest->getSearchTerm()]);
@ -207,12 +202,7 @@ class Elasticsearch implements Singleton, ConnectionInterface
return $this->objectManager->get(SearchResult::class, $searchRequest, $search->search()); return $this->objectManager->get(SearchResult::class, $searchRequest, $search->search());
} }
/** protected function getType(string $documentType) : \Elastica\Type
* @param string $documentType
*
* @return \Elastica\Type
*/
protected function getType($documentType)
{ {
return $this->typeFactory->getType( return $this->typeFactory->getType(
$this->indexFactory->getIndex( $this->indexFactory->getIndex(

View file

@ -44,7 +44,7 @@ class Connection implements Singleton
/** /**
* @param ConfigurationContainerInterface $configuration * @param ConfigurationContainerInterface $configuration
* @param \Elastica\Client $elasticaClient * @param \Elastica\Client|null $elasticaClient
*/ */
public function __construct( public function __construct(
ConfigurationContainerInterface $configuration, ConfigurationContainerInterface $configuration,
@ -52,9 +52,8 @@ class Connection implements Singleton
) { ) {
$this->configuration = $configuration; $this->configuration = $configuration;
$this->elasticaClient = $elasticaClient; if ($elasticaClient === null) {
if ($this->elasticaClient === null) { $elasticaClient = new \Elastica\Client([
$this->elasticaClient = new \Elastica\Client([
'host' => $this->configuration->get('connections.elasticsearch.host'), 'host' => $this->configuration->get('connections.elasticsearch.host'),
'port' => $this->configuration->get('connections.elasticsearch.port'), 'port' => $this->configuration->get('connections.elasticsearch.port'),
// TODO: Make configurable // TODO: Make configurable
@ -63,14 +62,13 @@ class Connection implements Singleton
// TODO: Make configurable. // TODO: Make configurable.
// new \Elastica\Log($this->elasticaClient); // new \Elastica\Log($this->elasticaClient);
} }
$this->elasticaClient = $elasticaClient;
} }
/** /**
* Get the concrete client for internal usage! * Get the concrete client for internal usage!
*
* @return \Elastica\Client
*/ */
public function getClient() public function getClient() : \Elastica\Client
{ {
return $this->elasticaClient; return $this->elasticaClient;
} }

View file

@ -44,13 +44,8 @@ class DocumentFactory implements Singleton
/** /**
* Creates document from document. * Creates document from document.
*
* @param string $documentType
* @param array $document
*
* @return \Elastica\Document
*/ */
public function getDocument($documentType, array $document) public function getDocument(string $documentType, array $document) : \Elastica\Document
{ {
// TODO: Use DocumentType for further configuration. // TODO: Use DocumentType for further configuration.
@ -70,13 +65,8 @@ class DocumentFactory implements Singleton
/** /**
* Creates documents based on documents. * Creates documents based on documents.
*
* @param string $documentType
* @param array $documents
*
* @return array
*/ */
public function getDocuments($documentType, array $documents) public function getDocuments(string $documentType, array $documents) : array
{ {
foreach ($documents as &$document) { foreach ($documents as &$document) {
$document = $this->getDocument($documentType, $document); $document = $this->getDocument($documentType, $document);

View file

@ -22,6 +22,7 @@ namespace Codappix\SearchCore\Connection\Elasticsearch;
use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
use Codappix\SearchCore\Connection\FacetInterface; use Codappix\SearchCore\Connection\FacetInterface;
use Codappix\SearchCore\Connection\FacetOptionInterface;
class Facet implements FacetInterface class Facet implements FacetInterface
{ {
@ -45,25 +46,26 @@ class Facet implements FacetInterface
*/ */
protected $options = []; protected $options = [];
public function __construct($name, array $aggregation, ConfigurationContainerInterface $configuration) public function __construct(string $name, array $aggregation, ConfigurationContainerInterface $configuration)
{ {
$this->name = $name; $this->name = $name;
$this->buckets = $aggregation['buckets']; $this->buckets = $aggregation['buckets'];
$this->field = $configuration->getIfExists('searching.facets.' . $this->name . '.field') ?: '';
$config = $configuration->getIfExists('searching.facets.' . $this->name) ?: [];
foreach ($config as $configEntry) {
if (isset($configEntry['field'])) {
$this->field = $configEntry['field'];
break;
}
}
} }
/** public function getName() : string
* @return string
*/
public function getName()
{ {
return $this->name; return $this->name;
} }
/** public function getField() : string
* @return string
*/
public function getField()
{ {
return $this->field; return $this->field;
} }
@ -73,7 +75,7 @@ class Facet implements FacetInterface
* *
* @return array<FacetOptionInterface> * @return array<FacetOptionInterface>
*/ */
public function getOptions() public function getOptions() : array
{ {
$this->initOptions(); $this->initOptions();

View file

@ -29,6 +29,11 @@ class FacetOption implements FacetOptionInterface
*/ */
protected $name = ''; protected $name = '';
/**
* @var string
*/
protected $displayName = '';
/** /**
* @var int * @var int
*/ */
@ -40,21 +45,21 @@ class FacetOption implements FacetOptionInterface
public function __construct(array $bucket) public function __construct(array $bucket)
{ {
$this->name = $bucket['key']; $this->name = $bucket['key'];
$this->displayName = isset($bucket['key_as_string']) ? $bucket['key_as_string'] : $this->getName();
$this->count = $bucket['doc_count']; $this->count = $bucket['doc_count'];
} }
/** public function getName() : string
* @return string
*/
public function getName()
{ {
return $this->name; return $this->name;
} }
/** public function getDisplayName() : string
* @return int {
*/ return $this->displayName;
public function getCount() }
public function getCount() : int
{ {
return $this->count; return $this->count;
} }

View file

@ -22,10 +22,8 @@ namespace Codappix\SearchCore\Connection\Elasticsearch;
use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Configuration\InvalidArgumentException;
use Elastica\Exception\ResponseException;
use TYPO3\CMS\Core\SingletonInterface as Singleton; use TYPO3\CMS\Core\SingletonInterface as Singleton;
use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Extbase\Configuration\ConfigurationManagerInterface;
/** /**
* Factory to get indexes. * Factory to get indexes.
@ -39,6 +37,21 @@ class IndexFactory implements Singleton
*/ */
protected $configuration; protected $configuration;
/**
* @var \TYPO3\CMS\Core\Log\Logger
*/
protected $logger;
/**
* Inject log manager to get concrete logger from it.
*
* @param \TYPO3\CMS\Core\Log\LogManager $logManager
*/
public function injectLogger(\TYPO3\CMS\Core\Log\LogManager $logManager)
{
$this->logger = $logManager->getLogger(__CLASS__);
}
/** /**
* @param ConfigurationContainerInterface $configuration * @param ConfigurationContainerInterface $configuration
*/ */
@ -59,36 +72,33 @@ class IndexFactory implements Singleton
/** /**
* Get an index bases on TYPO3 table name. * Get an index bases on TYPO3 table name.
*
* @param Connection $connection
* @param string $documentType
*
* @return \Elastica\Index
*/ */
public function getIndex(Connection $connection, $documentType) public function getIndex(Connection $connection, string $documentType) : \Elastica\Index
{ {
$index = $connection->getClient()->getIndex($this->getIndexName()); $index = $connection->getClient()->getIndex($this->getIndexName());
if ($index->exists() === false) { if ($index->exists() === false) {
$index->create($this->getConfigurationFor($documentType)); $config = $this->getConfigurationFor($documentType);
$this->logger->debug(sprintf('Create index %s.', $documentType), [$documentType, $config]);
$index->create($config);
$this->logger->debug(sprintf('Created index %s.', $documentType), [$documentType]);
} }
return $index; return $index;
} }
/** protected function getConfigurationFor(string $documentType) : array
* @param string $documentType
*
* @return array
*/
protected function getConfigurationFor($documentType)
{ {
try { try {
$configuration = $this->configuration->get('indexing.' . $documentType . '.index'); $configuration = $this->configuration->get('indexing.' . $documentType . '.index');
if (isset($configuration['analysis']['analyzer'])) { foreach (['analyzer', 'filter'] as $optionsToExpand) {
foreach ($configuration['analysis']['analyzer'] as $key => $analyzer) { if (isset($configuration['analysis'][$optionsToExpand])) {
$configuration['analysis']['analyzer'][$key] = $this->prepareAnalyzerConfiguration($analyzer); foreach ($configuration['analysis'][$optionsToExpand] as $key => $options) {
$configuration['analysis'][$optionsToExpand][$key] = $this->prepareAnalyzerConfiguration(
$options
);
}
} }
} }
@ -98,14 +108,9 @@ class IndexFactory implements Singleton
} }
} }
/** protected function prepareAnalyzerConfiguration(array $analyzer) : array
* @param array $analyzer
*
* @return array
*/
protected function prepareAnalyzerConfiguration(array $analyzer)
{ {
$fieldsToExplode = ['char_filter', 'filter']; $fieldsToExplode = ['char_filter', 'filter', 'word_list'];
foreach ($fieldsToExplode as $fieldToExplode) { foreach ($fieldsToExplode as $fieldToExplode) {
if (isset($analyzer[$fieldToExplode])) { if (isset($analyzer[$fieldToExplode])) {

View file

@ -44,12 +44,8 @@ class MappingFactory implements Singleton
/** /**
* Get an mapping based on type. * Get an mapping based on type.
*
* @param \Elastica\Type $type
*
* @return \Elastica\Mapping
*/ */
public function getMapping(\Elastica\Type $type) public function getMapping(\Elastica\Type $type) : \Elastica\Type\Mapping
{ {
$mapping = new \Elastica\Type\Mapping(); $mapping = new \Elastica\Type\Mapping();
$mapping->setType($type); $mapping->setType($type);
@ -64,11 +60,7 @@ class MappingFactory implements Singleton
return $mapping; return $mapping;
} }
/** protected function getConfiguration(string $identifier) : array
* @param string $identifier
* @return array
*/
protected function getConfiguration($identifier)
{ {
try { try {
return $this->configuration->get('indexing.' . $identifier . '.mapping'); return $this->configuration->get('indexing.' . $identifier . '.mapping');

View file

@ -24,10 +24,14 @@ use Codappix\SearchCore\Connection\FacetInterface;
use Codappix\SearchCore\Connection\ResultItemInterface; use Codappix\SearchCore\Connection\ResultItemInterface;
use Codappix\SearchCore\Connection\SearchRequestInterface; use Codappix\SearchCore\Connection\SearchRequestInterface;
use Codappix\SearchCore\Connection\SearchResultInterface; use Codappix\SearchCore\Connection\SearchResultInterface;
use Codappix\SearchCore\Domain\Model\QueryResultInterfaceStub;
use Codappix\SearchCore\Domain\Model\ResultItem;
use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; use TYPO3\CMS\Extbase\Object\ObjectManagerInterface;
class SearchResult implements SearchResultInterface class SearchResult implements SearchResultInterface
{ {
use QueryResultInterfaceStub;
/** /**
* @var SearchRequestInterface * @var SearchRequestInterface
*/ */
@ -73,7 +77,7 @@ class SearchResult implements SearchResultInterface
/** /**
* @return array<ResultItemInterface> * @return array<ResultItemInterface>
*/ */
public function getResults() public function getResults() : array
{ {
$this->initResults(); $this->initResults();
@ -85,14 +89,14 @@ class SearchResult implements SearchResultInterface
* *
* @return array<FacetInterface> * @return array<FacetInterface>
*/ */
public function getFacets() public function getFacets() : array
{ {
$this->initFacets(); $this->initFacets();
return $this->facets; return $this->facets;
} }
public function getCurrentCount() public function getCurrentCount() : int
{ {
return $this->result->count(); return $this->result->count();
} }
@ -104,7 +108,7 @@ class SearchResult implements SearchResultInterface
} }
foreach ($this->result->getResults() as $result) { foreach ($this->result->getResults() as $result) {
$this->results[] = new ResultItem($result); $this->results[] = new ResultItem($result->getData(), $result->getParam('_type'));
} }
} }
@ -153,41 +157,8 @@ class SearchResult implements SearchResultInterface
$this->position = 0; $this->position = 0;
} }
// Extbase QueryResultInterface - Implemented to support Pagination of Fluid.
public function getQuery() public function getQuery()
{ {
return $this->searchRequest; return $this->searchRequest;
} }
public function getFirst()
{
throw new \BadMethodCallException('Method is not implemented yet.', 1502195121);
}
public function toArray()
{
throw new \BadMethodCallException('Method is not implemented yet.', 1502195135);
}
public function offsetExists($offset)
{
// Return false to allow Fluid to use appropriate getter methods.
return false;
}
public function offsetGet($offset)
{
throw new \BadMethodCallException('Use getter to fetch properties.', 1502196933);
}
public function offsetSet($offset, $value)
{
throw new \BadMethodCallException('You are not allowed to modify the result.', 1502196934);
}
public function offsetUnset($offset)
{
throw new \BadMethodCallException('You are not allowed to modify the result.', 1502196936);
}
} }

View file

@ -21,7 +21,6 @@ namespace Codappix\SearchCore\Connection\Elasticsearch;
*/ */
use TYPO3\CMS\Core\SingletonInterface as Singleton; use TYPO3\CMS\Core\SingletonInterface as Singleton;
use TYPO3\CMS\Extbase\Configuration\ConfigurationManagerInterface;
/** /**
* Factory to get indexes. * Factory to get indexes.
@ -32,13 +31,8 @@ class TypeFactory implements Singleton
{ {
/** /**
* Get an index bases on TYPO3 table name. * Get an index bases on TYPO3 table name.
*
* @param \Elastica\Index $index
* @param string $documentType
*
* @return \Elastica\Type
*/ */
public function getType(\Elastica\Index $index, $documentType) public function getType(\Elastica\Index $index, string $documentType) : \Elastica\Type
{ {
return $index->getType($documentType); return $index->getType($documentType);
} }

View file

@ -25,15 +25,12 @@ namespace Codappix\SearchCore\Connection;
*/ */
interface FacetInterface interface FacetInterface
{ {
/** public function getName() : string;
* @return string
*/
public function getName();
/** /**
* Returns all possible options for this facet. * Returns all possible options for this facet.
* *
* @return array<FacetOptionInterface> * @return array<FacetOptionInterface>
*/ */
public function getOptions(); public function getOptions() : array;
} }

View file

@ -28,15 +28,17 @@ interface FacetOptionInterface
/** /**
* Returns the name of this option. Equivalent * Returns the name of this option. Equivalent
* to value used for filtering. * to value used for filtering.
*
* @return string
*/ */
public function getName(); public function getName() : string;
/**
* If a pre-rendered name is provided, this will be returned.
* Otherwise it's the same as getName().
*/
public function getDisplayName() : string;
/** /**
* Returns the number of found results for this option. * Returns the number of found results for this option.
*
* @return int
*/ */
public function getCount(); public function getCount() : int;
} }

View file

@ -28,15 +28,11 @@ interface FacetRequestInterface
/** /**
* The identifier of the facet, used as key in arrays and to get the facet * The identifier of the facet, used as key in arrays and to get the facet
* from search request, etc. * from search request, etc.
*
* @return string
*/ */
public function getIdentifier(); public function getIdentifier() : string;
/** /**
* The field to use for facet building. * The config to use for facet building.
*
* @return string
*/ */
public function getField(); public function getConfig() : array;
} }

View file

@ -25,5 +25,20 @@ namespace Codappix\SearchCore\Connection;
*/ */
interface ResultItemInterface extends \ArrayAccess interface ResultItemInterface extends \ArrayAccess
{ {
/**
* Returns every information as array.
*
* Provide key/column/field => data.
*
* Used e.g. for dataprocessing.
*/
public function getPlainData() : array;
/**
* Returns the type of the item.
*
* That should make it easier to differentiate if multiple
* types are returned for one query.
*/
public function getType() : string;
} }

View file

@ -20,24 +20,47 @@ namespace Codappix\SearchCore\Connection;
* 02110-1301, USA. * 02110-1301, USA.
*/ */
use Codappix\SearchCore\Connection\ConnectionInterface;
use Codappix\SearchCore\Connection\FacetRequestInterface;
use Codappix\SearchCore\Domain\Search\SearchService;
use TYPO3\CMS\Extbase\Persistence\QueryInterface; use TYPO3\CMS\Extbase\Persistence\QueryInterface;
interface SearchRequestInterface extends QueryInterface interface SearchRequestInterface extends QueryInterface
{ {
/** /**
* Returns the actual string the user searched for. * Returns the actual string the user searched for.
*/
public function getSearchTerm() : string;
public function hasFilter() : bool;
public function getFilter() : array;
public function setFilter(array $filter);
/**
* @return void
*/
public function addFacet(FacetRequestInterface $facet);
/**
* @return array<FacetRequestInterface>
*/
public function getFacets() : array;
/**
* Workaround for paginate widget support which will
* use the request to build another search.
* *
* @return string * @return void
*/ */
public function getSearchTerm(); public function setConnection(ConnectionInterface $connection);
/** /**
* @return bool * Workaround for paginate widget support which will
* use the request to build another search.
*
* @return void
*/ */
public function hasFilter(); public function setSearchService(SearchService $searchService);
/**
* @return array
*/
public function getFilter();
} }

View file

@ -30,19 +30,17 @@ interface SearchResultInterface extends \Iterator, \Countable, QueryResultInterf
/** /**
* @return array<ResultItemInterface> * @return array<ResultItemInterface>
*/ */
public function getResults(); public function getResults() : array;
/** /**
* Return all facets, if any. * Return all facets, if any.
* *
* @return array<FacetInterface> * @return array<FacetInterface>
*/ */
public function getFacets(); public function getFacets() : array;
/** /**
* Returns the number of results in current result * Returns the number of results in current result
*
* @return int
*/ */
public function getCurrentCount(); public function getCurrentCount() : int;
} }

View file

@ -0,0 +1,59 @@
<?php
namespace Codappix\SearchCore\DataProcessing;
/*
* Copyright (C) 2017 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
use Codappix\SearchCore\Compatibility\TypoScriptServiceInterface;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Frontend\ContentObject\ContentObjectRenderer;
/**
* Executes an existing TYPO3 DataProcessor on the given data.
*/
class ContentObjectDataProcessorAdapterProcessor implements ProcessorInterface
{
/**
* @var TypoScriptServiceInterface
*/
protected $typoScriptService;
public function __construct(TypoScriptServiceInterface $typoScriptService)
{
$this->typoScriptService = $typoScriptService;
}
public function processData(array $data, array $configuration) : array
{
$dataProcessor = GeneralUtility::makeInstance($configuration['_dataProcessor']);
$contentObjectRenderer = GeneralUtility::makeInstance(ContentObjectRenderer::class);
$contentObjectRenderer->data = $data;
if (isset($configuration['_table'])) {
$contentObjectRenderer->start($data, $configuration['_table']);
}
return $dataProcessor->process(
$contentObjectRenderer,
[],
$this->typoScriptService->convertPlainArrayToTypoScriptArray($configuration),
$data
);
}
}

View file

@ -25,13 +25,23 @@ namespace Codappix\SearchCore\DataProcessing;
*/ */
class CopyToProcessor implements ProcessorInterface class CopyToProcessor implements ProcessorInterface
{ {
public function processRecord(array $record, array $configuration) : array public function processData(array $record, array $configuration) : array
{ {
$all = []; $target = [];
$this->addArray($all, $record); $from = $record;
$all = array_filter($all); if (isset($configuration['from'])) {
$record[$configuration['to']] = implode(PHP_EOL, $all); $from = $record[$configuration['from']];
}
if (is_array($from)) {
$this->addArray($target, $from);
} else {
$target[] = (string) $from;
}
$target = array_filter($target);
$record[$configuration['to']] = implode(PHP_EOL, $target);
return $record; return $record;
} }

View file

@ -25,9 +25,9 @@ namespace Codappix\SearchCore\DataProcessing;
*/ */
class GeoPointProcessor implements ProcessorInterface class GeoPointProcessor implements ProcessorInterface
{ {
public function processRecord(array $record, array $configuration) : array public function processData(array $record, array $configuration) : array
{ {
if (! $this->canApply($record, $configuration)) { if (! $this->isApplyable($record, $configuration)) {
return $record; return $record;
} }
@ -39,7 +39,7 @@ class GeoPointProcessor implements ProcessorInterface
return $record; return $record;
} }
protected function canApply(array $record, array $configuration) : bool protected function isApplyable(array $record, array $configuration) : bool
{ {
if (!isset($record[$configuration['lat']]) if (!isset($record[$configuration['lat']])
|| !is_numeric($record[$configuration['lat']]) || !is_numeric($record[$configuration['lat']])

View file

@ -21,14 +21,13 @@ namespace Codappix\SearchCore\DataProcessing;
*/ */
/** /**
* All DataProcessing Processors should implement this interface, otherwise they * All DataProcessing Processors should implement this interface.
* will not be executed.
*/ */
interface ProcessorInterface interface ProcessorInterface
{ {
/** /**
* Processes the given record. * Processes the given data.
* Also retrieves the configuration for this processor instance. * Also retrieves the configuration for this processor instance.
*/ */
public function processRecord(array $record, array $configuration) : array; public function processData(array $record, array $configuration) : array;
} }

View file

@ -27,7 +27,7 @@ use TYPO3\CMS\Core\Utility\GeneralUtility;
*/ */
class RemoveProcessor implements ProcessorInterface class RemoveProcessor implements ProcessorInterface
{ {
public function processRecord(array $record, array $configuration) : array public function processData(array $record, array $configuration) : array
{ {
if (!isset($configuration['fields'])) { if (!isset($configuration['fields'])) {
return $record; return $record;

View file

@ -0,0 +1,60 @@
<?php
namespace Codappix\SearchCore\DataProcessing;
/*
* Copyright (C) 2018 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
use TYPO3\CMS\Extbase\Object\ObjectManagerInterface;
/**
* Eases work with data processing.
*/
class Service
{
/**
* @var ObjectManagerInterface
*/
protected $objectManager;
public function __construct(ObjectManagerInterface $objectManager)
{
$this->objectManager = $objectManager;
}
/**
* Executes the dataprocessor depending on configuration and returns the result.
*
* @param array|string $configuration Either the full configuration or only the class name.
*/
public function executeDataProcessor($configuration, array $data, string $recordType = '') : array
{
if (is_string($configuration)) {
$configuration = [
'_typoScriptNodeValue' => $configuration,
];
}
if (!isset($configuration['_table']) && $recordType !== '') {
$configuration['_table'] = $recordType;
}
return $this->objectManager->get($configuration['_typoScriptNodeValue'])
->processData($data, $configuration);
}
}

View file

@ -0,0 +1,101 @@
<?php
namespace Codappix\SearchCore\DataProcessing;
/*
* Copyright (C) 2018 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
use Codappix\SearchCore\Domain\Index\TcaIndexer\RelationResolver;
use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Extbase\Object\ObjectManagerInterface;
/**
* Resolves relations from TCA using RelationResolver.
*/
class TcaRelationResolvingProcessor implements ProcessorInterface
{
/**
* @var ObjectManagerInterface
*/
protected $objectManager;
/**
* @var RelationResolver
*/
protected $relationResolver;
public function __construct(
ObjectManagerInterface $objectManager,
RelationResolver $relationResolver
) {
$this->objectManager = $objectManager;
$this->relationResolver = $relationResolver;
}
/**
* @throws \InvalidArgumentException If _table is not configured.
*/
public function processData(array $record, array $configuration) : array
{
$this->initializeConfiguration($configuration);
$tcaTableService = $this->objectManager->get(
TcaTableServiceInterface::class,
$configuration['_table']
);
$processedRecord = $this->relationResolver->resolveRelationsForRecord(
$tcaTableService,
$this->getRecordToProcess($record, $configuration)
);
return array_merge($record, $processedRecord);
}
/**
* @throws \InvalidArgumentException If _table is not configured.
*/
protected function initializeConfiguration(array &$configuration)
{
if (!isset($configuration['_table'])) {
throw new \InvalidArgumentException('The configuration "_table" is mandantory.', 1524552631);
}
if (!isset($configuration['excludeFields'])) {
$configuration['excludeFields'] = '';
}
$configuration['excludeFields'] = GeneralUtility::trimExplode(',', $configuration['excludeFields'], true);
}
protected function getRecordToProcess(array $record, array $configuration) : array
{
if ($configuration['excludeFields'] === []) {
return $record;
}
$newRecord = [];
$keysToUse = array_diff(array_keys($record), $configuration['excludeFields']);
foreach ($keysToUse as $keyToUse) {
$newRecord[$keyToUse] = $record[$keyToUse];
}
return $newRecord;
}
}

View file

@ -23,7 +23,6 @@ namespace Codappix\SearchCore\Domain\Index;
use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Configuration\InvalidArgumentException;
use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Connection\ConnectionInterface;
use Codappix\SearchCore\DataProcessing\ProcessorInterface;
use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Core\Utility\GeneralUtility;
abstract class AbstractIndexer implements IndexerInterface abstract class AbstractIndexer implements IndexerInterface
@ -43,6 +42,12 @@ abstract class AbstractIndexer implements IndexerInterface
*/ */
protected $identifier = ''; protected $identifier = '';
/**
* @var \Codappix\SearchCore\DataProcessing\Service
* @inject
*/
protected $dataProcessorService;
/** /**
* @var \TYPO3\CMS\Core\Log\Logger * @var \TYPO3\CMS\Core\Log\Logger
*/ */
@ -58,15 +63,11 @@ abstract class AbstractIndexer implements IndexerInterface
$this->logger = $logManager->getLogger(__CLASS__); $this->logger = $logManager->getLogger(__CLASS__);
} }
public function setIdentifier($identifier) public function setIdentifier(string $identifier)
{ {
$this->identifier = $identifier; $this->identifier = $identifier;
} }
/**
* @param ConnectionInterface $connection
* @param ConfigurationContainerInterface $configuration
*/
public function __construct(ConnectionInterface $connection, ConfigurationContainerInterface $configuration) public function __construct(ConnectionInterface $connection, ConfigurationContainerInterface $configuration)
{ {
$this->connection = $connection; $this->connection = $connection;
@ -91,11 +92,11 @@ abstract class AbstractIndexer implements IndexerInterface
$this->logger->info('Finish indexing'); $this->logger->info('Finish indexing');
} }
public function indexDocument($identifier) public function indexDocument(string $identifier)
{ {
$this->logger->info('Start indexing single record.', [$identifier]); $this->logger->info('Start indexing single record.', [$identifier]);
try { try {
$record = $this->getRecord($identifier); $record = $this->getRecord((int) $identifier);
$this->prepareRecord($record); $this->prepareRecord($record);
$this->connection->addDocument($this->getDocumentName(), $record); $this->connection->addDocument($this->getDocumentName(), $record);
@ -113,10 +114,7 @@ abstract class AbstractIndexer implements IndexerInterface
$this->logger->info('Finish deletion.'); $this->logger->info('Finish deletion.');
} }
/** protected function getRecordGenerator() : \Generator
* @return \Generator
*/
protected function getRecordGenerator()
{ {
$offset = 0; $offset = 0;
$limit = $this->getLimit(); $limit = $this->getLimit();
@ -127,24 +125,11 @@ abstract class AbstractIndexer implements IndexerInterface
} }
} }
/**
* @param array &$record
*/
protected function prepareRecord(array &$record) protected function prepareRecord(array &$record)
{ {
try { try {
foreach ($this->configuration->get('indexing.' . $this->identifier . '.dataProcessing') as $configuration) { foreach ($this->configuration->get('indexing.' . $this->identifier . '.dataProcessing') as $configuration) {
$className = ''; $record = $this->dataProcessorService->executeDataProcessor($configuration, $record, $this->identifier);
if (is_string($configuration)) {
$className = $configuration;
$configuration = [];
} else {
$className = $configuration['_typoScriptNodeValue'];
}
$dataProcessor = GeneralUtility::makeInstance($className);
if ($dataProcessor instanceof ProcessorInterface) {
$record = $dataProcessor->processRecord($record, $configuration);
}
} }
} catch (InvalidArgumentException $e) { } catch (InvalidArgumentException $e) {
// Nothing to do. // Nothing to do.
@ -153,9 +138,6 @@ abstract class AbstractIndexer implements IndexerInterface
$this->handleAbstract($record); $this->handleAbstract($record);
} }
/**
* @param array &$record
*/
protected function handleAbstract(array &$record) protected function handleAbstract(array &$record)
{ {
$record['search_abstract'] = ''; $record['search_abstract'] = '';
@ -165,7 +147,7 @@ abstract class AbstractIndexer implements IndexerInterface
',', ',',
$this->configuration->get('indexing.' . $this->identifier . '.abstractFields') $this->configuration->get('indexing.' . $this->identifier . '.abstractFields')
); );
if (!$fieldsToUse) { if ($fieldsToUse === []) {
return; return;
} }
foreach ($fieldsToUse as $fieldToUse) { foreach ($fieldsToUse as $fieldToUse) {
@ -181,31 +163,22 @@ abstract class AbstractIndexer implements IndexerInterface
/** /**
* Returns the limit to use to fetch records. * Returns the limit to use to fetch records.
*
* @return int
*/ */
protected function getLimit() protected function getLimit() : int
{ {
// TODO: Make configurable. // TODO: Make configurable.
return 50; return 50;
} }
/** /**
* @param int $offset
* @param int $limit
* @return array|null * @return array|null
*/ */
abstract protected function getRecords($offset, $limit); abstract protected function getRecords(int $offset, int $limit);
/** /**
* @param int $identifier
* @return array
* @throws NoRecordFoundException If record could not be found. * @throws NoRecordFoundException If record could not be found.
*/ */
abstract protected function getRecord($identifier); abstract protected function getRecord(int $identifier) : array;
/** abstract protected function getDocumentName() : string;
* @return string
*/
abstract protected function getDocumentName();
} }

View file

@ -23,8 +23,7 @@ namespace Codappix\SearchCore\Domain\Index;
use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Configuration\InvalidArgumentException;
use Codappix\SearchCore\Domain\Index\IndexerInterface; use Codappix\SearchCore\Domain\Index\IndexerInterface;
use Codappix\SearchCore\Domain\Index\TcaIndexer; use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface;
use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService;
use TYPO3\CMS\Core\SingletonInterface as Singleton; use TYPO3\CMS\Core\SingletonInterface as Singleton;
use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; use TYPO3\CMS\Extbase\Object\ObjectManagerInterface;
@ -56,12 +55,9 @@ class IndexerFactory implements Singleton
} }
/** /**
* @param string $identifier
*
* @return IndexerInterface
* @throws NoMatchingIndexer * @throws NoMatchingIndexer
*/ */
public function getIndexer($identifier) public function getIndexer(string $identifier) : IndexerInterface
{ {
try { try {
return $this->buildIndexer($this->configuration->get('indexing.' . $identifier . '.indexer'), $identifier); return $this->buildIndexer($this->configuration->get('indexing.' . $identifier . '.indexer'), $identifier);
@ -75,13 +71,9 @@ class IndexerFactory implements Singleton
} }
/** /**
* @param string $indexerClass
* @param string $identifier
*
* @return IndexerInterface
* @throws NoMatchingIndexer * @throws NoMatchingIndexer
*/ */
protected function buildIndexer($indexerClass, $identifier) protected function buildIndexer(string $indexerClass, string $identifier) : IndexerInterface
{ {
$indexer = null; $indexer = null;
if (is_subclass_of($indexerClass, TcaIndexer\PagesIndexer::class) if (is_subclass_of($indexerClass, TcaIndexer\PagesIndexer::class)
@ -89,13 +81,13 @@ class IndexerFactory implements Singleton
) { ) {
$indexer = $this->objectManager->get( $indexer = $this->objectManager->get(
$indexerClass, $indexerClass,
$this->objectManager->get(TcaTableService::class, $identifier), $this->objectManager->get(TcaTableServiceInterface::class, $identifier),
$this->objectManager->get(TcaTableService::class, 'tt_content') $this->objectManager->get(TcaTableServiceInterface::class, 'tt_content')
); );
} elseif (is_subclass_of($indexerClass, TcaIndexer::class) || $indexerClass === TcaIndexer::class) { } elseif (is_subclass_of($indexerClass, TcaIndexer::class) || $indexerClass === TcaIndexer::class) {
$indexer = $this->objectManager->get( $indexer = $this->objectManager->get(
$indexerClass, $indexerClass,
$this->objectManager->get(TcaTableService::class, $identifier) $this->objectManager->get(TcaTableServiceInterface::class, $identifier)
); );
} elseif (class_exists($indexerClass) && in_array(IndexerInterface::class, class_implements($indexerClass))) { } elseif (class_exists($indexerClass) && in_array(IndexerInterface::class, class_implements($indexerClass))) {
$indexer = $this->objectManager->get($indexerClass); $indexer = $this->objectManager->get($indexerClass);

View file

@ -35,20 +35,16 @@ interface IndexerInterface
/** /**
* Fetches a single document and pushes it to the connection. * Fetches a single document and pushes it to the connection.
* *
* @param string $identifier
*
* @return void * @return void
*/ */
public function indexDocument($identifier); public function indexDocument(string $identifier);
/** /**
* Recieves the identifier of the indexer itself. * Recieves the identifier of the indexer itself.
* *
* @param string $identifier
*
* @return void * @return void
*/ */
public function setIdentifier($identifier); public function setIdentifier(string $identifier);
/** /**
* Delete the whole index. * Delete the whole index.

View file

@ -22,9 +22,7 @@ namespace Codappix\SearchCore\Domain\Index;
use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Connection\ConnectionInterface;
use TYPO3\CMS\Core\Database\ConnectionPool; use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface;
use TYPO3\CMS\Core\Database\Query\QueryBuilder;
use TYPO3\CMS\Core\Utility\GeneralUtility;
/** /**
* Will index the given table using configuration from TCA. * Will index the given table using configuration from TCA.
@ -32,39 +30,31 @@ use TYPO3\CMS\Core\Utility\GeneralUtility;
class TcaIndexer extends AbstractIndexer class TcaIndexer extends AbstractIndexer
{ {
/** /**
* @var TcaIndexer\TcaTableService * @var TcaTableServiceInterface
*/ */
protected $tcaTableService; protected $tcaTableService;
/** /**
* @param TcaIndexer\TcaTableService $tcaTableService * @param TcaTableServiceInterface $tcaTableService
* @param ConnectionInterface $connection * @param ConnectionInterface $connection
* @param ConfigurationContainerInterface $configuration * @param ConfigurationContainerInterface $configuration
*/ */
public function __construct( public function __construct(
TcaIndexer\TcaTableService $tcaTableService, TcaTableServiceInterface $tcaTableService,
ConnectionInterface $connection, ConnectionInterface $connection,
ConfigurationContainerInterface $configuration ConfigurationContainerInterface $configuration
) { ) {
parent::__construct($connection, $configuration);
$this->tcaTableService = $tcaTableService; $this->tcaTableService = $tcaTableService;
$this->connection = $connection;
$this->configuration = $configuration;
} }
/** /**
* @param int $offset
* @param int $limit
* @return array|null * @return array|null
*/ */
protected function getRecords($offset, $limit) protected function getRecords(int $offset, int $limit)
{ {
$records = $this->getQuery() $records = $this->tcaTableService->getRecords($offset, $limit);
->setFirstResult($offset) if ($records === []) {
->setMaxResults($limit)
->execute()
->fetchAll();
if ($records === null) {
return null; return null;
} }
@ -77,17 +67,13 @@ class TcaIndexer extends AbstractIndexer
} }
/** /**
* @param int $identifier
* @return array
* @throws NoRecordFoundException If record could not be found. * @throws NoRecordFoundException If record could not be found.
*/ */
protected function getRecord($identifier) protected function getRecord(int $identifier) : array
{ {
$query = $this->getQuery(); $record = $this->tcaTableService->getRecord($identifier);
$query = $query->andWhere($this->tcaTableService->getTableName() . '.uid = ' . (int) $identifier);
$record = $query->execute()->fetch();
if ($record === false || $record === null) { if ($record === []) {
throw new NoRecordFoundException( throw new NoRecordFoundException(
'Record could not be fetched from database: "' . $identifier . '". Perhaps record is not active.', 'Record could not be fetched from database: "' . $identifier . '". Perhaps record is not active.',
1484225364 1484225364
@ -98,36 +84,8 @@ class TcaIndexer extends AbstractIndexer
return $record; return $record;
} }
/** protected function getDocumentName() : string
* @return string
*/
protected function getDocumentName()
{ {
return $this->tcaTableService->getTableName(); return $this->tcaTableService->getTableName();
} }
protected function getQuery($tcaTableService = null) : QueryBuilder
{
if ($tcaTableService === null) {
$tcaTableService = $this->tcaTableService;
}
$queryBuilder = $this->getDatabaseConnection()->getQueryBuilderForTable($tcaTableService->getTableName());
$where = $tcaTableService->getWhereClause();
$query = $queryBuilder->select(... $tcaTableService->getFields())
->from($tcaTableService->getTableClause())
->where($where->getStatement())
->setParameters($where->getParameters());
foreach ($tcaTableService->getJoins() as $join) {
$query->from($join->getTable());
$query->andWhere($join->getCondition());
}
return $query;
}
protected function getDatabaseConnection()
{
return GeneralUtility::makeInstance(ConnectionPool::class);
}
} }

View file

@ -23,6 +23,8 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer;
use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Connection\ConnectionInterface;
use Codappix\SearchCore\Domain\Index\TcaIndexer; use Codappix\SearchCore\Domain\Index\TcaIndexer;
use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService;
use TYPO3\CMS\Core\Utility\GeneralUtility;
/** /**
* Specific indexer for Pages, will basically add content of page. * Specific indexer for Pages, will basically add content of page.
@ -30,33 +32,36 @@ use Codappix\SearchCore\Domain\Index\TcaIndexer;
class PagesIndexer extends TcaIndexer class PagesIndexer extends TcaIndexer
{ {
/** /**
* @var TcaTableService * @var TcaTableServiceInterface
*/ */
protected $contentTableService; protected $contentTableService;
/** /**
* @param TcaTableService $tcaTableService * @var \TYPO3\CMS\Core\Resource\FileRepository
* @param TcaTableService $tcaTableService * @inject
*/
protected $fileRepository;
/**
* @param TcaTableServiceInterface $tcaTableService
* @param TcaTableServiceInterface $contentTableService
* @param ConnectionInterface $connection * @param ConnectionInterface $connection
* @param ConfigurationContainerInterface $configuration * @param ConfigurationContainerInterface $configuration
*/ */
public function __construct( public function __construct(
TcaTableService $tcaTableService, TcaTableServiceInterface $tcaTableService,
TcaTableService $contentTableService, TcaTableServiceInterface $contentTableService,
ConnectionInterface $connection, ConnectionInterface $connection,
ConfigurationContainerInterface $configuration ConfigurationContainerInterface $configuration
) { ) {
$this->tcaTableService = $tcaTableService; parent::__construct($tcaTableService, $connection, $configuration);
$this->contentTableService = $contentTableService; $this->contentTableService = $contentTableService;
$this->connection = $connection;
$this->configuration = $configuration;
} }
/**
* @param array &$record
*/
protected function prepareRecord(array &$record) protected function prepareRecord(array &$record)
{ {
parent::prepareRecord($record);
$possibleTitleFields = ['nav_title', 'tx_tqseo_pagetitle_rel', 'title']; $possibleTitleFields = ['nav_title', 'tx_tqseo_pagetitle_rel', 'title'];
foreach ($possibleTitleFields as $searchTitleField) { foreach ($possibleTitleFields as $searchTitleField) {
if (isset($record[$searchTitleField]) && trim($record[$searchTitleField])) { if (isset($record[$searchTitleField]) && trim($record[$searchTitleField])) {
@ -65,32 +70,96 @@ class PagesIndexer extends TcaIndexer
} }
} }
$record['content'] = $this->fetchContentForPage($record['uid']); $record['media'] = $this->fetchMediaForPage($record['uid']);
parent::prepareRecord($record); $content = $this->fetchContentForPage($record['uid']);
if ($content !== []) {
$record['content'] = $content['content'];
$record['media'] = array_values(array_unique(array_merge($record['media'], $content['images'])));
}
} }
/** protected function fetchContentForPage(int $uid) : array
* @param int $uid
* @return string
*/
protected function fetchContentForPage($uid)
{ {
$contentElements = $this->getQuery($this->contentTableService)->execute()->fetchAll(); if ($this->contentTableService instanceof TcaTableService) {
$queryBuilder = $this->contentTableService->getQuery();
$queryBuilder->andWhere(
$queryBuilder->expr()->eq(
$this->contentTableService->getTableName() . '.pid',
$queryBuilder->createNamedParameter($uid, \PDO::PARAM_INT)
)
);
$contentElements = $queryBuilder->execute()->fetchAll();
} else {
$contentElements = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows(
$this->contentTableService->getFields(),
$this->contentTableService->getTableClause(),
$this->contentTableService->getWhereClause() .
sprintf(' AND %s.pid = %u', $this->contentTableService->getTableName(), $uid)
);
}
if ($contentElements === null) { if ($contentElements === null) {
$this->logger->debug('No content for page ' . $uid); $this->logger->debug('No content for page ' . $uid);
return ''; return [];
} }
$this->logger->debug('Fetched content for page ' . $uid); $this->logger->debug('Fetched content for page ' . $uid);
$images = [];
$content = []; $content = [];
foreach ($contentElements as $contentElement) { foreach ($contentElements as $contentElement) {
$content[] = $contentElement['bodytext']; $images = array_merge(
$images,
$this->getContentElementImages($contentElement['uid'])
);
$content[] = $this->getContentFromContentElement($contentElement);
} }
// Remove Tags. return [
// Interpret escaped new lines and special chars. // Remove Tags.
// Trim, e.g. trailing or leading new lines. // Interpret escaped new lines and special chars.
return trim(stripcslashes(strip_tags(implode(' ', $content)))); // Trim, e.g. trailing or leading new lines.
'content' => trim(stripcslashes(strip_tags(implode(' ', $content)))),
'images' => $images,
];
}
protected function getContentElementImages(int $uidOfContentElement) : array
{
return $this->fetchSysFileReferenceUids($uidOfContentElement, 'tt_content', 'image');
}
protected function fetchMediaForPage(int $uid) : array
{
return $this->fetchSysFileReferenceUids($uid, 'pages', 'media');
}
protected function fetchSysFileReferenceUids(int $uid, string $tablename, string $fieldname) : array
{
$imageRelationUids = [];
$imageRelations = $this->fileRepository->findByRelation($tablename, $fieldname, $uid);
foreach ($imageRelations as $relation) {
$imageRelationUids[] = $relation->getUid();
}
return $imageRelationUids;
}
protected function getContentFromContentElement(array $contentElement) : string
{
$content = '';
$fieldsWithContent = GeneralUtility::trimExplode(
',',
$this->configuration->get('indexing.' . $this->identifier . '.contentFields'),
true
);
foreach ($fieldsWithContent as $fieldWithContent) {
if (isset($contentElement[$fieldWithContent]) && trim($contentElement[$fieldWithContent])) {
$content .= trim($contentElement[$fieldWithContent]) . ' ';
}
}
return trim($content);
} }
} }

View file

@ -33,11 +33,11 @@ use TYPO3\CMS\Core\Utility\GeneralUtility;
*/ */
class RelationResolver implements Singleton class RelationResolver implements Singleton
{ {
public function resolveRelationsForRecord(TcaTableService $service, array &$record) : void public function resolveRelationsForRecord(TcaTableServiceInterface $service, array $record) : array
{ {
foreach (array_keys($record) as $column) { foreach (array_keys($record) as $column) {
// TODO: Define / configure fields to exclude?! if (in_array($column, ['pid', $service->getLanguageUidColumn()])) {
if ($column === 'pid') { $record[$column] = (int) $record[$column];
continue; continue;
} }
@ -61,6 +61,8 @@ class RelationResolver implements Singleton
continue; continue;
} }
} }
return $record;
} }
protected function resolveValue($value, array $tcaColumn) protected function resolveValue($value, array $tcaColumn)
@ -82,7 +84,7 @@ class RelationResolver implements Singleton
protected function isRelation(array &$config) : bool protected function isRelation(array &$config) : bool
{ {
return isset($config['foreign_table']) return isset($config['foreign_table'])
|| (isset($config['renderType']) && $config['renderType'] !== 'selectSingle') || (isset($config['renderType']) && !in_array($config['renderType'], ['selectSingle', 'inputDateTime']))
|| (isset($config['internal_type']) && strtolower($config['internal_type']) === 'db') || (isset($config['internal_type']) && strtolower($config['internal_type']) === 'db')
; ;
} }
@ -97,7 +99,7 @@ class RelationResolver implements Singleton
return array_map('trim', explode(',', $value)); return array_map('trim', explode(',', $value));
} }
protected function getUtilityForMode(): string protected function getUtilityForMode() : string
{ {
if (TYPO3_MODE === 'BE') { if (TYPO3_MODE === 'BE') {
return BackendUtility::class; return BackendUtility::class;

View file

@ -21,11 +21,13 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer;
*/ */
use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
use Codappix\SearchCore\Configuration\InvalidArgumentException as InvalidConfigurationArgumentException;
use Codappix\SearchCore\Database\Doctrine\Join; use Codappix\SearchCore\Database\Doctrine\Join;
use Codappix\SearchCore\Database\Doctrine\Where; use Codappix\SearchCore\Database\Doctrine\Where;
use Codappix\SearchCore\Domain\Index\IndexingException; use Codappix\SearchCore\Domain\Index\IndexingException;
use Codappix\SearchCore\Domain\Index\TcaIndexer\InvalidArgumentException;
use TYPO3\CMS\Backend\Utility\BackendUtility; use TYPO3\CMS\Backend\Utility\BackendUtility;
use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\Database\Query\QueryBuilder;
use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Core\Utility\RootlineUtility; use TYPO3\CMS\Core\Utility\RootlineUtility;
use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; use TYPO3\CMS\Extbase\Object\ObjectManagerInterface;
@ -33,7 +35,7 @@ use TYPO3\CMS\Extbase\Object\ObjectManagerInterface;
/** /**
* Encapsulate logik related to TCA configuration. * Encapsulate logik related to TCA configuration.
*/ */
class TcaTableService class TcaTableService implements TcaTableServiceInterface
{ {
/** /**
* TCA for current table. * TCA for current table.
@ -52,11 +54,6 @@ class TcaTableService
*/ */
protected $configuration; protected $configuration;
/**
* @var RelationResolver
*/
protected $relationResolver;
/** /**
* @var \TYPO3\CMS\Core\Log\Logger * @var \TYPO3\CMS\Core\Log\Logger
*/ */
@ -91,7 +88,6 @@ class TcaTableService
*/ */
public function __construct( public function __construct(
$tableName, $tableName,
RelationResolver $relationResolver,
ConfigurationContainerInterface $configuration ConfigurationContainerInterface $configuration
) { ) {
if (!isset($GLOBALS['TCA'][$tableName])) { if (!isset($GLOBALS['TCA'][$tableName])) {
@ -104,32 +100,39 @@ class TcaTableService
$this->tableName = $tableName; $this->tableName = $tableName;
$this->tca = &$GLOBALS['TCA'][$this->tableName]; $this->tca = &$GLOBALS['TCA'][$this->tableName];
$this->configuration = $configuration; $this->configuration = $configuration;
$this->relationResolver = $relationResolver;
} }
/**
* @return string
*/
public function getTableName() : string public function getTableName() : string
{ {
return $this->tableName; return $this->tableName;
} }
/**
* @return string
*/
public function getTableClause() : string public function getTableClause() : string
{ {
return $this->tableName; return $this->tableName;
} }
/** public function getRecords(int $offset, int $limit) : array
* Filter the given records by root line blacklist settings. {
* $records = $this->getQuery()
* @param array &$records ->setFirstResult($offset)
* @return void ->setMaxResults($limit)
*/ ->execute()
public function filterRecordsByRootLineBlacklist(array &$records) : void ->fetchAll();
return $records ?: [];
}
public function getRecord(int $identifier) : array
{
$query = $this->getQuery();
$query = $query->andWhere($this->getTableName() . '.uid = ' . $identifier);
$record = $query->execute()->fetch();
return $record ?: [];
}
public function filterRecordsByRootLineBlacklist(array &$records)
{ {
$records = array_filter( $records = array_filter(
$records, $records,
@ -139,13 +142,8 @@ class TcaTableService
); );
} }
/** public function prepareRecord(array &$record)
* @param array &$record
*/
public function prepareRecord(array &$record) : void
{ {
$this->relationResolver->resolveRelationsForRecord($this, $record);
if (isset($record['uid']) && !isset($record['search_identifier'])) { if (isset($record['uid']) && !isset($record['search_identifier'])) {
$record['search_identifier'] = $record['uid']; $record['search_identifier'] = $record['uid'];
} }
@ -154,18 +152,20 @@ class TcaTableService
} }
} }
public function getWhereClause() : Where protected function getWhereClause() : Where
{ {
$parameters = []; $parameters = [];
$whereClause = $this->getSystemWhereClause(); $whereClause = $this->getSystemWhereClause();
$userDefinedWhere = $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.additionalWhereClause'); $userDefinedWhere = $this->configuration->getIfExists(
'indexing.' . $this->getTableName() . '.additionalWhereClause'
);
if (is_string($userDefinedWhere)) { if (is_string($userDefinedWhere)) {
$whereClause .= ' AND ' . $userDefinedWhere; $whereClause .= ' AND ' . $userDefinedWhere;
} }
if ($this->isBlacklistedRootLineConfigured()) { if ($this->isBlackListedRootLineConfigured()) {
$parameters[':blacklistedRootLine'] = $this->getBlacklistedRootLine(); $parameters[':blacklistedRootLine'] = implode(',', $this->getBlackListedRootLine());
$whereClause .= ' AND pages.uid NOT IN (:blacklistedRootLine)' $whereClause .= ' AND pages.uid NOT IN (:blacklistedRootLine)'
. ' AND pages.pid NOT IN (:blacklistedRootLine)'; . ' AND pages.pid NOT IN (:blacklistedRootLine)';
} }
@ -174,7 +174,7 @@ class TcaTableService
return new Where($whereClause, $parameters); return new Where($whereClause, $parameters);
} }
public function getFields() : array protected function getFields() : array
{ {
$fields = array_merge( $fields = array_merge(
['uid','pid'], ['uid','pid'],
@ -197,7 +197,7 @@ class TcaTableService
return $fields; return $fields;
} }
public function getJoins() : array protected function getJoins() : array
{ {
if ($this->tableName === 'pages') { if ($this->tableName === 'pages') {
return []; return [];
@ -212,7 +212,7 @@ class TcaTableService
* Generate SQL for TYPO3 as a system, to make sure only available records * Generate SQL for TYPO3 as a system, to make sure only available records
* are fetched. * are fetched.
*/ */
public function getSystemWhereClause() : string protected function getSystemWhereClause() : string
{ {
$whereClause = '1=1' $whereClause = '1=1'
. BackendUtility::BEenableFields($this->tableName) . BackendUtility::BEenableFields($this->tableName)
@ -229,11 +229,7 @@ class TcaTableService
return $whereClause; return $whereClause;
} }
/** protected function isSystemField(string $columnName) : bool
* @param string
* @return bool
*/
protected function isSystemField($columnName) : bool
{ {
$systemFields = [ $systemFields = [
// Versioning fields, // Versioning fields,
@ -245,7 +241,6 @@ class TcaTableService
$this->tca['ctrl']['cruser_id'], $this->tca['ctrl']['cruser_id'],
$this->tca['ctrl']['fe_cruser_id'], $this->tca['ctrl']['fe_cruser_id'],
$this->tca['ctrl']['fe_crgroup_id'], $this->tca['ctrl']['fe_crgroup_id'],
$this->tca['ctrl']['languageField'],
$this->tca['ctrl']['origUid'], $this->tca['ctrl']['origUid'],
]; ];
@ -265,11 +260,9 @@ class TcaTableService
} }
/** /**
* @param string $columnName
* @return array
* @throws InvalidArgumentException * @throws InvalidArgumentException
*/ */
public function getColumnConfig($columnName) : array public function getColumnConfig(string $columnName) : array
{ {
if (!isset($this->tca['columns'][$columnName])) { if (!isset($this->tca['columns'][$columnName])) {
throw new InvalidArgumentException( throw new InvalidArgumentException(
@ -281,6 +274,15 @@ class TcaTableService
return $this->tca['columns'][$columnName]['config']; return $this->tca['columns'][$columnName]['config'];
} }
public function getLanguageUidColumn() : string
{
if (!isset($this->tca['ctrl']['languageField'])) {
return '';
}
return $this->tca['ctrl']['languageField'];
}
/** /**
* Checks whether the given record was blacklisted by root line. * Checks whether the given record was blacklisted by root line.
* This can be configured by typoscript as whole root lines can be black listed. * This can be configured by typoscript as whole root lines can be black listed.
@ -288,9 +290,6 @@ class TcaTableService
* Also further TYPO3 mechanics are taken into account. Does a valid root * Also further TYPO3 mechanics are taken into account. Does a valid root
* line exist, is page inside a recycler, is inherited start- endtime * line exist, is page inside a recycler, is inherited start- endtime
* excluded, etc. * excluded, etc.
*
* @param array &$record
* @return bool
*/ */
protected function isRecordBlacklistedByRootline(array &$record) : bool protected function isRecordBlacklistedByRootline(array &$record) : bool
{ {
@ -346,8 +345,6 @@ class TcaTableService
/** /**
* Checks whether any page uids are black listed. * Checks whether any page uids are black listed.
*
* @return bool
*/ */
protected function isBlackListedRootLineConfigured() : bool protected function isBlackListedRootLineConfigured() : bool
{ {
@ -361,6 +358,31 @@ class TcaTableService
*/ */
protected function getBlackListedRootLine() : array protected function getBlackListedRootLine() : array
{ {
return GeneralUtility::intExplode(',', $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist')); return GeneralUtility::intExplode(
',',
$this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist')
);
}
public function getQuery() : QueryBuilder
{
$queryBuilder = $this->getDatabaseConnection()->getQueryBuilderForTable($this->getTableName());
$where = $this->getWhereClause();
$query = $queryBuilder->select(... $this->getFields())
->from($this->getTableClause())
->where($where->getStatement())
->setParameters($where->getParameters());
foreach ($this->getJoins() as $join) {
$query->from($join->getTable());
$query->andWhere($join->getCondition());
}
return $query;
}
protected function getDatabaseConnection() : ConnectionPool
{
return GeneralUtility::makeInstance(ConnectionPool::class);
} }
} }

View file

@ -0,0 +1,378 @@
<?php
namespace Codappix\SearchCore\Domain\Index\TcaIndexer;
/*
* Copyright (C) 2016 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
use Codappix\SearchCore\Domain\Index\IndexingException;
use Codappix\SearchCore\Domain\Index\TcaIndexer\InvalidArgumentException;
use TYPO3\CMS\Backend\Utility\BackendUtility;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Core\Utility\RootlineUtility;
use TYPO3\CMS\Extbase\Object\ObjectManagerInterface;
/**
* Encapsulate logik related to TCA configuration.
*/
class TcaTableService76 implements TcaTableServiceInterface
{
/**
* TCA for current table.
* !REFERENCE! To save memory.
* @var array
*/
protected $tca;
/**
* @var string
*/
protected $tableName;
/**
* @var ConfigurationContainerInterface
*/
protected $configuration;
/**
* @var \TYPO3\CMS\Core\Log\Logger
*/
protected $logger;
/**
* @var ObjectManagerInterface
*/
protected $objectManager;
/**
* Inject log manager to get concrete logger from it.
*
* @param \TYPO3\CMS\Core\Log\LogManager $logManager
*/
public function injectLogger(\TYPO3\CMS\Core\Log\LogManager $logManager)
{
$this->logger = $logManager->getLogger(__CLASS__);
}
/**
* @param ObjectManagerInterface $objectManager
*/
public function injectObjectManager(ObjectManagerInterface $objectManager)
{
$this->objectManager = $objectManager;
}
/**
* @param string $tableName
* @param ConfigurationContainerInterface $configuration
*/
public function __construct(
$tableName,
ConfigurationContainerInterface $configuration
) {
if (!isset($GLOBALS['TCA'][$tableName])) {
throw new IndexingException(
'Table "' . $tableName . '" is not configured in TCA.',
IndexingException::CODE_UNKOWN_TCA_TABLE
);
}
$this->tableName = $tableName;
$this->tca = &$GLOBALS['TCA'][$this->tableName];
$this->configuration = $configuration;
}
public function getTableName() : string
{
return $this->tableName;
}
public function getTableClause() : string
{
if ($this->tableName === 'pages') {
return $this->tableName;
}
return $this->tableName . ' LEFT JOIN pages on ' . $this->tableName . '.pid = pages.uid';
}
public function getRecords(int $offset, int $limit) : array
{
$records = $this->getConnection()->exec_SELECTgetRows(
$this->getFields(),
$this->getTableClause(),
$this->getWhereClause(),
'',
'',
(int) $offset . ',' . (int) $limit
);
return $records ?: [];
}
public function getRecord(int $identifier) : array
{
$record = $this->getConnection()->exec_SELECTgetSingleRow(
$this->getFields(),
$this->getTableClause(),
$this->getWhereClause()
. ' AND ' . $this->getTableName() . '.uid = ' . (int) $identifier
);
return $record ?: [];
}
public function filterRecordsByRootLineBlacklist(array &$records)
{
$records = array_filter(
$records,
function ($record) {
return ! $this->isRecordBlacklistedByRootline($record);
}
);
}
public function prepareRecord(array &$record)
{
if (isset($record['uid']) && !isset($record['search_identifier'])) {
$record['search_identifier'] = $record['uid'];
}
if (isset($record[$this->tca['ctrl']['label']]) && !isset($record['search_title'])) {
$record['search_title'] = $record[$this->tca['ctrl']['label']];
}
}
public function getWhereClause() : string
{
$whereClause = '1=1'
. BackendUtility::BEenableFields($this->tableName)
. BackendUtility::deleteClause($this->tableName)
. ' AND pages.no_search = 0'
;
if ($this->tableName !== 'pages') {
$whereClause .= BackendUtility::BEenableFields('pages')
. BackendUtility::deleteClause('pages')
;
}
$userDefinedWhere = $this->configuration->getIfExists(
'indexing.' . $this->getTableName() . '.additionalWhereClause'
);
if (is_string($userDefinedWhere)) {
$whereClause .= ' AND ' . $userDefinedWhere;
}
if ($this->isBlacklistedRootLineConfigured()) {
$whereClause .= ' AND pages.uid NOT IN ('
. implode(',', $this->getBlacklistedRootLine())
. ')'
. ' AND pages.pid NOT IN ('
. implode(',', $this->getBlacklistedRootLine())
. ')';
}
$this->logger->debug('Generated where clause.', [$this->tableName, $whereClause]);
return $whereClause;
}
public function getFields() : string
{
$fields = array_merge(
['uid','pid'],
array_filter(
array_keys($this->tca['columns']),
function ($columnName) {
return !$this->isSystemField($columnName)
&& !$this->isUserField($columnName)
&& !$this->isPassthroughField($columnName)
;
}
)
);
foreach ($fields as $key => $field) {
$fields[$key] = $this->tableName . '.' . $field;
}
$this->logger->debug('Generated fields.', [$this->tableName, $fields]);
return implode(',', $fields);
}
/**
* Generate SQL for TYPO3 as a system, to make sure only available records
* are fetched.
*/
protected function getSystemWhereClause() : string
{
$whereClause = '1=1'
. BackendUtility::BEenableFields($this->tableName)
. BackendUtility::deleteClause($this->tableName)
. ' AND pages.no_search = 0'
;
if ($this->tableName !== 'pages') {
$whereClause .= BackendUtility::BEenableFields('pages')
. BackendUtility::deleteClause('pages')
;
}
return $whereClause;
}
protected function isSystemField(string $columnName) : bool
{
$systemFields = [
// Versioning fields,
// https://docs.typo3.org/typo3cms/TCAReference/Reference/Ctrl/Index.html#versioningws
't3ver_oid', 't3ver_id', 't3ver_label', 't3ver_wsid',
't3ver_state', 't3ver_stage', 't3ver_count', 't3ver_tstamp',
't3ver_move_id', 't3ver_swapmode',
$this->tca['ctrl']['transOrigDiffSourceField'],
$this->tca['ctrl']['cruser_id'],
$this->tca['ctrl']['fe_cruser_id'],
$this->tca['ctrl']['fe_crgroup_id'],
$this->tca['ctrl']['origUid'],
];
return in_array($columnName, $systemFields);
}
protected function isUserField(string $columnName) : bool
{
$config = $this->getColumnConfig($columnName);
return isset($config['type']) && $config['type'] === 'user';
}
protected function isPassthroughField(string $columnName) : bool
{
$config = $this->getColumnConfig($columnName);
return isset($config['type']) && $config['type'] === 'passthrough';
}
/**
* @throws InvalidArgumentException
*/
public function getColumnConfig(string $columnName) : array
{
if (!isset($this->tca['columns'][$columnName])) {
throw new InvalidArgumentException(
'Column does not exist.',
InvalidArgumentException::COLUMN_DOES_NOT_EXIST
);
}
return $this->tca['columns'][$columnName]['config'];
}
public function getLanguageUidColumn() : string
{
if (!isset($this->tca['ctrl']['languageField'])) {
return '';
}
return $this->tca['ctrl']['languageField'];
}
/**
* Checks whether the given record was blacklisted by root line.
* This can be configured by typoscript as whole root lines can be black listed.
*
* Also further TYPO3 mechanics are taken into account. Does a valid root
* line exist, is page inside a recycler, is inherited start- endtime
* excluded, etc.
*/
protected function isRecordBlacklistedByRootline(array &$record) : bool
{
$pageUid = $record['pid'];
if ($this->tableName === 'pages') {
$pageUid = $record['uid'];
}
try {
$rootline = $this->objectManager->get(RootlineUtility::class, $pageUid)->get();
} catch (\RuntimeException $e) {
$this->logger->notice(
sprintf('Could not fetch rootline for page %u, because: %s', $pageUid, $e->getMessage()),
[$record, $e]
);
return true;
}
foreach ($rootline as $pageInRootLine) {
// Check configured black list if present.
if ($this->isBlackListedRootLineConfigured()
&& in_array($pageInRootLine['uid'], $this->getBlackListedRootLine())
) {
$this->logger->info(
sprintf(
'Record %u is black listed due to configured root line configuration of page %u.',
$record['uid'],
$pageInRootLine['uid']
),
[$record, $pageInRootLine]
);
return true;
}
if ($pageInRootLine['extendToSubpages'] && (
($pageInRootLine['endtime'] > 0 && $pageInRootLine['endtime'] <= time())
|| ($pageInRootLine['starttime'] > 0 && $pageInRootLine['starttime'] >= time())
)) {
$this->logger->info(
sprintf(
'Record %u is black listed due to configured timing of parent page %u.',
$record['uid'],
$pageInRootLine['uid']
),
[$record, $pageInRootLine]
);
return true;
}
}
return false;
}
/**
* Checks whether any page uids are black listed.
*/
protected function isBlackListedRootLineConfigured() : bool
{
return (bool) $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist');
}
/**
* Get the list of black listed root line page uids.
*
* @return array<Int>
*/
protected function getBlackListedRootLine() : array
{
return GeneralUtility::intExplode(
',',
$this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist')
);
}
protected function getConnection() : \TYPO3\CMS\Core\Database\DatabaseConnection
{
return $GLOBALS['TYPO3_DB'];
}
}

View file

@ -0,0 +1,46 @@
<?php
namespace Codappix\SearchCore\Domain\Index\TcaIndexer;
/*
* Copyright (C) 2016 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
interface TcaTableServiceInterface
{
public function getTableName() : string;
public function getTableClause() : string;
/**
* Filter the given records by root line blacklist settings.
*/
public function filterRecordsByRootLineBlacklist(array &$records);
public function prepareRecord(array &$record);
/**
* @throws InvalidArgumentException
*/
public function getColumnConfig(string $columnName) : array;
public function getRecords(int $offset, int $limit) : array;
public function getRecord(int $identifier) : array;
public function getLanguageUidColumn() : string;
}

View file

@ -30,37 +30,27 @@ class FacetRequest implements FacetRequestInterface
protected $identifier = ''; protected $identifier = '';
/** /**
* @var string * @var array
*/ */
protected $field = ''; protected $config = [];
/** /**
* TODO: Add validation / exception?
* As the facets come from configuration this might be a good idea to help * As the facets come from configuration this might be a good idea to help
* integrators find issues. * integrators find issues.
*
* @param string $identifier
* @param string $field
*/ */
public function __construct($identifier, $field) public function __construct(string $identifier, array $config)
{ {
$this->identifier = $identifier; $this->identifier = $identifier;
$this->field = $field; $this->config = $config;
} }
/** public function getIdentifier() : string
* @return string
*/
public function getIdentifier()
{ {
return $this->identifier; return $this->identifier;
} }
/** public function getConfig() : array
* @return string
*/
public function getField()
{ {
return $this->field; return $this->config;
} }
} }

View file

@ -0,0 +1,61 @@
<?php
namespace Codappix\SearchCore\Domain\Model;
/*
* Copyright (C) 2018 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
/**
* As we have to stay compatible with QueryResultInterface
* of extbase but can and need not to provide all methods,
* this stub will provde the non implemented methods to
* keep real implementations clean.
*/
trait QueryResultInterfaceStub
{
public function getFirst()
{
throw new \BadMethodCallException('Method is not implemented yet.', 1502195121);
}
public function toArray()
{
throw new \BadMethodCallException('Method is not implemented yet.', 1502195135);
}
public function offsetExists($offset)
{
// Return false to allow Fluid to use appropriate getter methods.
return false;
}
public function offsetGet($offset)
{
throw new \BadMethodCallException('Use getter to fetch properties.', 1502196933);
}
public function offsetSet($offset, $value)
{
throw new \BadMethodCallException('You are not allowed to modify the result.', 1502196934);
}
public function offsetUnset($offset)
{
throw new \BadMethodCallException('You are not allowed to modify the result.', 1502196936);
}
}

View file

@ -1,5 +1,5 @@
<?php <?php
namespace Codappix\SearchCore\Connection\Elasticsearch; namespace Codappix\SearchCore\Domain\Model;
/* /*
* Copyright (C) 2017 Daniel Siepmann <coding@daniel-siepmann.de> * Copyright (C) 2017 Daniel Siepmann <coding@daniel-siepmann.de>
@ -29,9 +29,25 @@ class ResultItem implements ResultItemInterface
*/ */
protected $data = []; protected $data = [];
public function __construct(\Elastica\Result $result) /**
* @var string
*/
protected $type = '';
public function __construct(array $result, string $type)
{ {
$this->data = $result->getData(); $this->data = $result;
$this->type = $type;
}
public function getType() : string
{
return $this->type;
}
public function getPlainData() : array
{
return $this->data;
} }
public function offsetExists($offset) public function offsetExists($offset)

View file

@ -23,6 +23,7 @@ namespace Codappix\SearchCore\Domain\Model;
use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Connection\ConnectionInterface;
use Codappix\SearchCore\Connection\FacetRequestInterface; use Codappix\SearchCore\Connection\FacetRequestInterface;
use Codappix\SearchCore\Connection\SearchRequestInterface; use Codappix\SearchCore\Connection\SearchRequestInterface;
use Codappix\SearchCore\Domain\Search\SearchService;
/** /**
* Represents a search request used to process an actual search. * Represents a search request used to process an actual search.
@ -64,25 +65,24 @@ class SearchRequest implements SearchRequestInterface
protected $connection = null; protected $connection = null;
/** /**
* @param string $query * @var SearchService
*/ */
public function __construct($query = '') protected $searchService = null;
{
$this->query = (string) $query;
}
/** /**
* @return string * @param string $query
*/ */
public function getQuery() public function __construct(string $query = '')
{
$this->query = $query;
}
public function getQuery() : string
{ {
return $this->query; return $this->query;
} }
/** public function getSearchTerm() : string
* @return string
*/
public function getSearchTerm()
{ {
return $this->query; return $this->query;
} }
@ -96,26 +96,18 @@ class SearchRequest implements SearchRequestInterface
$this->filter = \TYPO3\CMS\Extbase\Utility\ArrayUtility::removeEmptyElementsRecursively($filter); $this->filter = \TYPO3\CMS\Extbase\Utility\ArrayUtility::removeEmptyElementsRecursively($filter);
} }
/** public function hasFilter() : bool
* @return bool
*/
public function hasFilter()
{ {
return count($this->filter) > 0; return count($this->filter) > 0;
} }
/** public function getFilter() : array
* @return array
*/
public function getFilter()
{ {
return $this->filter; return $this->filter;
} }
/** /**
* Add a facet to gather in this search request. * Add a facet to gather in this search request.
*
* @param FacetRequestInterface $facet
*/ */
public function addFacet(FacetRequestInterface $facet) public function addFacet(FacetRequestInterface $facet)
{ {
@ -124,10 +116,8 @@ class SearchRequest implements SearchRequestInterface
/** /**
* Returns all configured facets to fetch in this search request. * Returns all configured facets to fetch in this search request.
*
* @return array
*/ */
public function getFacets() public function getFacets() : array
{ {
return $this->facets; return $this->facets;
} }
@ -135,36 +125,49 @@ class SearchRequest implements SearchRequestInterface
/** /**
* Define connection to use for this request. * Define connection to use for this request.
* Necessary to allow implementation of execute for interface. * Necessary to allow implementation of execute for interface.
*
* @param ConnectionInterface $connection
*/ */
public function setConnection(ConnectionInterface $connection) public function setConnection(ConnectionInterface $connection)
{ {
$this->connection = $connection; $this->connection = $connection;
} }
public function setSearchService(SearchService $searchService)
{
$this->searchService = $searchService;
}
// Extbase QueryInterface // Extbase QueryInterface
// Current implementation covers only paginate widget support. // Current implementation covers only paginate widget support.
public function execute($returnRawQueryResult = false) public function execute($returnRawQueryResult = false)
{ {
if ($this->connection instanceof ConnectionInterface) { if (! ($this->connection instanceof ConnectionInterface)) {
return $this->connection->search($this); throw new \InvalidArgumentException(
'Connection was not set before, therefore execute can not work. Use `setConnection` before.',
1502197732
);
}
if (! ($this->searchService instanceof SearchService)) {
throw new \InvalidArgumentException(
'SearchService was not set before, therefore execute can not work. Use `setSearchService` before.',
1520325175
);
} }
throw new \InvalidArgumentException( return $this->searchService->processResult($this->connection->search($this));
'Connection was not set before, therefore execute can not work. Use `setConnection` before.',
1502197732
);
} }
public function setLimit($limit) public function setLimit($limit)
{ {
$this->limit = (int) $limit; $this->limit = (int) $limit;
return $this;
} }
public function setOffset($offset) public function setOffset($offset)
{ {
$this->offset = (int) $offset; $this->offset = (int) $offset;
return $this;
} }
public function getLimit() public function getLimit()

View file

@ -0,0 +1,129 @@
<?php
namespace Codappix\SearchCore\Domain\Model;
/*
* Copyright (C) 2016 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
use Codappix\SearchCore\Connection\ResultItemInterface;
use Codappix\SearchCore\Connection\SearchResultInterface;
use Codappix\SearchCore\Domain\Model\QueryResultInterfaceStub;
/**
* Generic model for mapping a concrete search result from a connection.
*/
class SearchResult implements SearchResultInterface
{
use QueryResultInterfaceStub;
/**
* @var SearchResultInterface
*/
protected $originalSearchResult;
/**
* @var array
*/
protected $resultItems = [];
/**
* @var array
*/
protected $results = [];
/**
* For Iterator interface.
*
* @var int
*/
protected $position = 0;
public function __construct(SearchResultInterface $originalSearchResult, array $resultItems)
{
$this->originalSearchResult = $originalSearchResult;
$this->resultItems = $resultItems;
}
/**
* @return array<ResultItemInterface>
*/
public function getResults() : array
{
$this->initResults();
return $this->results;
}
protected function initResults()
{
if ($this->results !== []) {
return;
}
foreach ($this->resultItems as $item) {
$this->results[] = new ResultItem($item['data'], $item['type']);
}
}
public function getFacets() : array
{
return $this->originalSearchResult->getFacets();
}
public function getCurrentCount() : int
{
return $this->originalSearchResult->getCurrentCount();
}
public function count()
{
return $this->originalSearchResult->count();
}
public function current()
{
return $this->getResults()[$this->position];
}
public function next()
{
++$this->position;
return $this->current();
}
public function key()
{
return $this->position;
}
public function valid()
{
return isset($this->getResults()[$this->position]);
}
public function rewind()
{
$this->position = 0;
}
public function getQuery()
{
return $this->originalSearchResult->getQuery();
}
}

View file

@ -23,8 +23,6 @@ namespace Codappix\SearchCore\Domain\Search;
use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
use Codappix\SearchCore\Configuration\ConfigurationUtility; use Codappix\SearchCore\Configuration\ConfigurationUtility;
use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Configuration\InvalidArgumentException;
use Codappix\SearchCore\Connection\ConnectionInterface;
use Codappix\SearchCore\Connection\Elasticsearch\Query;
use Codappix\SearchCore\Connection\SearchRequestInterface; use Codappix\SearchCore\Connection\SearchRequestInterface;
use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Extbase\Utility\ArrayUtility; use TYPO3\CMS\Extbase\Utility\ArrayUtility;
@ -121,6 +119,10 @@ class QueryFactory
return; return;
} }
if (trim($searchRequest->getSearchTerm()) === '') {
return;
}
$boostQueryParts = []; $boostQueryParts = [];
foreach ($fields as $fieldName => $boostValue) { foreach ($fields as $fieldName => $boostValue) {
@ -134,13 +136,15 @@ class QueryFactory
]; ];
} }
$query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ if (!empty($boostQueryParts)) {
'query' => [ $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [
'bool' => [ 'query' => [
'should' => $boostQueryParts, 'bool' => [
'should' => $boostQueryParts,
],
], ],
], ]);
]); }
} }
protected function addFactorBoost(array &$query) protected function addFactorBoost(array &$query)
@ -161,7 +165,11 @@ class QueryFactory
{ {
try { try {
$query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [
'stored_fields' => GeneralUtility::trimExplode(',', $this->configuration->get('searching.fields.stored_fields'), true), 'stored_fields' => GeneralUtility::trimExplode(
',',
$this->configuration->get('searching.fields.stored_fields'),
true
),
]); ]);
} catch (InvalidArgumentException $e) { } catch (InvalidArgumentException $e) {
// Nothing configured // Nothing configured
@ -169,7 +177,10 @@ class QueryFactory
try { try {
$scriptFields = $this->configuration->get('searching.fields.script_fields'); $scriptFields = $this->configuration->get('searching.fields.script_fields');
$scriptFields = $this->configurationUtility->replaceArrayValuesWithRequestContent($searchRequest, $scriptFields); $scriptFields = $this->configurationUtility->replaceArrayValuesWithRequestContent(
$searchRequest,
$scriptFields
);
$scriptFields = $this->configurationUtility->filterByCondition($scriptFields); $scriptFields = $this->configurationUtility->filterByCondition($scriptFields);
if ($scriptFields !== []) { if ($scriptFields !== []) {
$query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['script_fields' => $scriptFields]); $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['script_fields' => $scriptFields]);
@ -231,6 +242,18 @@ class QueryFactory
} }
} }
if (isset($config['raw'])) {
$filter = array_merge($config['raw'], $filter);
}
if ($config['type'] === 'range') {
return [
'range' => [
$config['field'] => $filter,
],
];
}
return [$config['field'] => $filter]; return [$config['field'] => $filter];
} }
@ -239,11 +262,7 @@ class QueryFactory
foreach ($searchRequest->getFacets() as $facet) { foreach ($searchRequest->getFacets() as $facet) {
$query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [
'aggs' => [ 'aggs' => [
$facet->getIdentifier() => [ $facet->getIdentifier() => $facet->getConfig(),
'terms' => [
'field' => $facet->getField(),
],
],
], ],
]); ]);
} }

View file

@ -25,7 +25,9 @@ use Codappix\SearchCore\Configuration\InvalidArgumentException;
use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Connection\ConnectionInterface;
use Codappix\SearchCore\Connection\SearchRequestInterface; use Codappix\SearchCore\Connection\SearchRequestInterface;
use Codappix\SearchCore\Connection\SearchResultInterface; use Codappix\SearchCore\Connection\SearchResultInterface;
use Codappix\SearchCore\DataProcessing\Service as DataProcessorService;
use Codappix\SearchCore\Domain\Model\FacetRequest; use Codappix\SearchCore\Domain\Model\FacetRequest;
use Codappix\SearchCore\Domain\Model\SearchResult;
use TYPO3\CMS\Core\Utility\ArrayUtility; use TYPO3\CMS\Core\Utility\ArrayUtility;
use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; use TYPO3\CMS\Extbase\Object\ObjectManagerInterface;
@ -49,39 +51,44 @@ class SearchService
*/ */
protected $objectManager; protected $objectManager;
/**
* @var DataProcessorService
*/
protected $dataProcessorService;
/** /**
* @param ConnectionInterface $connection * @param ConnectionInterface $connection
* @param ConfigurationContainerInterface $configuration * @param ConfigurationContainerInterface $configuration
* @param ObjectManagerInterface $objectManager * @param ObjectManagerInterface $objectManager
* @param DataProcessorService $dataProcessorService
*/ */
public function __construct( public function __construct(
ConnectionInterface $connection, ConnectionInterface $connection,
ConfigurationContainerInterface $configuration, ConfigurationContainerInterface $configuration,
ObjectManagerInterface $objectManager ObjectManagerInterface $objectManager,
DataProcessorService $dataProcessorService
) { ) {
$this->connection = $connection; $this->connection = $connection;
$this->configuration = $configuration; $this->configuration = $configuration;
$this->objectManager = $objectManager; $this->objectManager = $objectManager;
$this->dataProcessorService = $dataProcessorService;
} }
/** public function search(SearchRequestInterface $searchRequest) : SearchResultInterface
* @param SearchRequestInterface $searchRequest
* @return SearchResultInterface
*/
public function search(SearchRequestInterface $searchRequest)
{ {
$searchRequest->setConnection($this->connection);
$this->addSize($searchRequest); $this->addSize($searchRequest);
$this->addConfiguredFacets($searchRequest); $this->addConfiguredFacets($searchRequest);
$this->addConfiguredFilters($searchRequest); $this->addConfiguredFilters($searchRequest);
return $this->connection->search($searchRequest); // Add connection to request to enable paginate widget support
$searchRequest->setConnection($this->connection);
$searchRequest->setSearchService($this);
return $this->processResult($this->connection->search($searchRequest));
} }
/** /**
* Add configured size of search result items to request. * Add configured size of search result items to request.
*
* @param SearchRequestInterface $searchRequest
*/ */
protected function addSize(SearchRequestInterface $searchRequest) protected function addSize(SearchRequestInterface $searchRequest)
{ {
@ -92,8 +99,6 @@ class SearchService
/** /**
* Add facets from configuration to request. * Add facets from configuration to request.
*
* @param SearchRequestInterface $searchRequest
*/ */
protected function addConfiguredFacets(SearchRequestInterface $searchRequest) protected function addConfiguredFacets(SearchRequestInterface $searchRequest)
{ {
@ -103,23 +108,16 @@ class SearchService
} }
foreach ($facetsConfig as $identifier => $facetConfig) { foreach ($facetsConfig as $identifier => $facetConfig) {
if (!isset($facetConfig['field']) || trim($facetConfig['field']) === '') {
// TODO: Finish throw
throw new \Exception('message', 1499171142);
}
$searchRequest->addFacet($this->objectManager->get( $searchRequest->addFacet($this->objectManager->get(
FacetRequest::class, FacetRequest::class,
$identifier, $identifier,
$facetConfig['field'] $facetConfig
)); ));
} }
} }
/** /**
* Add filters from configuration, e.g. flexform or TypoScript. * Add filters from configuration, e.g. flexform or TypoScript.
*
* @param SearchRequestInterface $searchRequest
*/ */
protected function addConfiguredFilters(SearchRequestInterface $searchRequest) protected function addConfiguredFilters(SearchRequestInterface $searchRequest)
{ {
@ -138,4 +136,33 @@ class SearchService
// Nothing todo, no filter configured. // Nothing todo, no filter configured.
} }
} }
/**
* Processes the result, e.g. applies configured data processing to result.
*/
public function processResult(SearchResultInterface $searchResult) : SearchResultInterface
{
try {
$newSearchResultItems = [];
foreach ($this->configuration->get('searching.dataProcessing') as $configuration) {
foreach ($searchResult as $resultItem) {
$newSearchResultItems[] = [
'data' => $this->dataProcessorService->executeDataProcessor(
$configuration,
$resultItem->getPlainData()
),
'type' => $resultItem->getType(),
];
}
}
return $this->objectManager->get(
SearchResult::class,
$searchResult,
$newSearchResultItems
);
} catch (InvalidArgumentException $e) {
return $searchResult;
}
}
} }

View file

@ -22,10 +22,9 @@ namespace Codappix\SearchCore\Domain\Service;
use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
use Codappix\SearchCore\Domain\Index\IndexerFactory; use Codappix\SearchCore\Domain\Index\IndexerFactory;
use Codappix\SearchCore\Domain\Index\IndexerInterface;
use Codappix\SearchCore\Domain\Index\NoMatchingIndexerException; use Codappix\SearchCore\Domain\Index\NoMatchingIndexerException;
use Codappix\SearchCore\Domain\Index\TcaIndexer;
use TYPO3\CMS\Core\SingletonInterface as Singleton; use TYPO3\CMS\Core\SingletonInterface as Singleton;
use TYPO3\CMS\Core\Utility\GeneralUtility;
/** /**
* Handles all data related things like updates, deletes and inserts. * Handles all data related things like updates, deletes and inserts.
@ -83,51 +82,27 @@ class DataHandler implements Singleton
$this->indexerFactory = $indexerFactory; $this->indexerFactory = $indexerFactory;
} }
/** public function update(string $table, array $record)
* @param string $table
* @param array $record
*/
public function add($table, array $record)
{
$this->logger->debug('Record received for add.', [$table, $record]);
$this->getIndexer($table)->indexDocument($record['uid']);
}
/**
* @param string $table
*/
public function update($table, array $record)
{ {
$this->logger->debug('Record received for update.', [$table, $record]); $this->logger->debug('Record received for update.', [$table, $record]);
$this->getIndexer($table)->indexDocument($record['uid']); $this->getIndexer($table)->indexDocument($record['uid']);
} }
/** public function delete(string $table, string $identifier)
* @param string $table
* @param int $identifier
*/
public function delete($table, $identifier)
{ {
$this->logger->debug('Record received for delete.', [$table, $identifier]); $this->logger->debug('Record received for delete.', [$table, $identifier]);
$this->connection->deleteDocument($table, $identifier); $this->connection->deleteDocument($table, $identifier);
} }
/** /**
* @param string $table
* @return IndexerInterface
*
* @throws NoMatchingIndexerException * @throws NoMatchingIndexerException
*/ */
protected function getIndexer($table) protected function getIndexer(string $table) : IndexerInterface
{ {
return $this->indexerFactory->getIndexer($table); return $this->indexerFactory->getIndexer($table);
} }
/** public function supportsTable(string $table) : bool
* @param string $table
* @return bool
*/
public function canHandle($table)
{ {
try { try {
$this->getIndexer($table); $this->getIndexer($table);
@ -135,7 +110,5 @@ class DataHandler implements Singleton
} catch (NoMatchingIndexerException $e) { } catch (NoMatchingIndexerException $e) {
return false; return false;
} }
return false;
} }
} }

View file

@ -21,7 +21,6 @@ namespace Codappix\SearchCore\Hook;
*/ */
use Codappix\SearchCore\Configuration\NoConfigurationException; use Codappix\SearchCore\Configuration\NoConfigurationException;
use Codappix\SearchCore\Domain\Index\NoMatchingIndexerException;
use Codappix\SearchCore\Domain\Service\DataHandler as OwnDataHandler; use Codappix\SearchCore\Domain\Service\DataHandler as OwnDataHandler;
use TYPO3\CMS\Backend\Utility\BackendUtility; use TYPO3\CMS\Backend\Utility\BackendUtility;
use TYPO3\CMS\Core\DataHandling\DataHandler as CoreDataHandler; use TYPO3\CMS\Core\DataHandling\DataHandler as CoreDataHandler;
@ -49,99 +48,103 @@ class DataHandler implements Singleton
/** /**
* Dependency injection as TYPO3 doesn't provide it on it's own. * Dependency injection as TYPO3 doesn't provide it on it's own.
* Still you can submit your own dataHandler. * Still you can submit your own dataHandler.
*
* @param OwnDataHandler $dataHandler
* @param Logger $logger
*/ */
public function __construct(OwnDataHandler $dataHandler = null, Logger $logger = null) public function __construct(OwnDataHandler $dataHandler = null, Logger $logger = null)
{ {
$this->dataHandler = $dataHandler; if ($dataHandler === null) {
if ($this->dataHandler === null) {
try { try {
$this->dataHandler = GeneralUtility::makeInstance(ObjectManager::class) $dataHandler = GeneralUtility::makeInstance(ObjectManager::class)
->get(OwnDataHandler::class); ->get(OwnDataHandler::class);
} catch (NoConfigurationException $e) { } catch (NoConfigurationException $e) {
// We have no configuration. That's fine, hooks will not be // We have no configuration. That's fine, hooks will not be
// executed due to check for existing DataHandler. // executed due to check for existing DataHandler.
} }
} }
$this->dataHandler = $dataHandler;
$this->logger = $logger; if ($logger === null) {
if ($this->logger === null) { $logger = GeneralUtility::makeInstance(LogManager::class)
$this->logger = GeneralUtility::makeInstance(LogManager::class)
->getLogger(__CLASS__); ->getLogger(__CLASS__);
} }
$this->logger = $logger;
} }
/** /**
* Called by CoreDataHandler on deletion of records. * Called by CoreDataHandler on deletion of records.
*
* @param string $table
* @param int $uid
*
* @return bool False if hook was not processed.
*/ */
public function processCmdmap_deleteAction($table, $uid) public function processCmdmap_deleteAction(string $table, string $uid) : bool
{ {
if (! $this->shouldProcessHookForTable($table)) { if (! $this->shouldProcessHookForTable($table)) {
$this->logger->debug('Delete not processed.', [$table, $uid]); $this->logger->debug('Delete not processed.', [$table, $uid]);
return false; return false;
} }
$this->dataHandler->delete($table, $uid); $this->dataHandler->delete($table, (string) $uid);
return true; return true;
} }
/** public function processDatamap_afterAllOperations(CoreDataHandler $dataHandler)
* Called by CoreDataHandler on database operations, e.g. if new records were created or records were updated. {
* foreach ($dataHandler->datamap as $table => $record) {
* @param string $status $uid = key($record);
* @param string $table $fieldData = current($record);
* @param string|int $uid
* @param array $fieldArray if (isset($fieldData['uid'])) {
* @param CoreDataHandler $dataHandler $uid = $fieldData['uid'];
* } elseif (isset($dataHandler->substNEWwithIDs[$uid])) {
* @return bool False if hook was not processed. $uid = $dataHandler->substNEWwithIDs[$uid];
*/ }
public function processDatamap_afterDatabaseOperations($status, $table, $uid, array $fieldArray, CoreDataHandler $dataHandler)
if (!is_numeric($uid) || $uid <= 0) {
continue;
}
$this->processRecord($table, $uid);
}
}
public function clearCachePostProc(array $parameters, CoreDataHandler $dataHandler)
{
$pageUid = 0;
// If editor uses "small page blizzard"
if (isset($parameters['cacheCmd']) && is_numeric($parameters['cacheCmd'])) {
$pageUid = $parameters['cacheCmd'];
}
// If records were changed
if (isset($parameters['uid_page']) && is_numeric($parameters['uid_page'])) {
$pageUid = $parameters['uid_page'];
}
if ($pageUid > 0) {
$this->processRecord('pages', (int) $pageUid);
}
}
protected function processRecord(string $table, int $uid) : bool
{ {
if (! $this->shouldProcessHookForTable($table)) { if (! $this->shouldProcessHookForTable($table)) {
$this->logger->debug('Database update not processed.', [$table, $uid]); $this->logger->debug('Indexing of record not processed.', [$table, $uid]);
return false; return false;
} }
if ($status === 'new') { $record = $this->getRecord($table, $uid);
$fieldArray['uid'] = $dataHandler->substNEWwithIDs[$uid]; if ($record !== null) {
$this->dataHandler->add($table, $fieldArray); $this->dataHandler->update($table, $record);
return true; return true;
} }
if ($status === 'update') { $this->logger->debug('Indexing of record not processed, as he was not found in Database.', [$table, $uid]);
$record = $this->getRecord($table, $uid);
if ($record !== null) {
$this->dataHandler->update($table, $record);
}
return true;
}
$this->logger->debug(
'Database update not processed, cause status is unhandled.',
[$status, $table, $uid, $fieldArray]
);
return false; return false;
} }
/** protected function shouldProcessHookForTable(string $table) : bool
* @param string $table
* @return bool
*/
protected function shouldProcessHookForTable($table)
{ {
if ($this->dataHandler === null) { if ($this->dataHandler === null) {
$this->logger->debug('Datahandler could not be setup.'); $this->logger->debug('Datahandler could not be setup.');
return false; return false;
} }
if (! $this->dataHandler->canHandle($table)) { if (! $this->dataHandler->supportsTable($table)) {
$this->logger->debug('Table is not allowed.', [$table]); $this->logger->debug('Table is not allowed.', [$table]);
return false; return false;
} }
@ -152,11 +155,9 @@ class DataHandler implements Singleton
/** /**
* Wrapper to allow unit testing. * Wrapper to allow unit testing.
* *
* @param string $table * @return array|null
* @param int $uid
* @return null|array<String>
*/ */
protected function getRecord($table, $uid) protected function getRecord(string $table, int $uid)
{ {
return BackendUtility::getRecord($table, $uid); return BackendUtility::getRecord($table, $uid);
} }

View file

@ -58,13 +58,11 @@ class DataHandlerFinisher extends AbstractFinisher
switch ($action) { switch ($action) {
case 'update': case 'update':
case 'add':
$this->dataHandler->update($tableName, $record); $this->dataHandler->update($tableName, $record);
break; break;
case 'add':
$this->dataHandler->add($tableName, $record);
break;
case 'delete': case 'delete':
$this->dataHandler->delete($tableName, $record['uid']); $this->dataHandler->delete($tableName, (string) $record['uid']);
break; break;
} }
} }

View file

@ -29,10 +29,7 @@ use TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController;
*/ */
class FrontendUtility extends BackendUtility class FrontendUtility extends BackendUtility
{ {
/** protected static function getLanguageService() : TypoScriptFrontendController
* @return TypoScriptFrontendController
*/
protected static function getLanguageService()
{ {
return $GLOBALS['TSFE']; return $GLOBALS['TSFE'];
} }

View file

@ -11,12 +11,13 @@ plugin {
indexing { indexing {
tt_content { tt_content {
additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu', 'shortcut', 'search', 'login') AND tt_content.bodytext != '' additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu', 'shortcut', 'search', 'login') AND (tt_content.bodytext != '' OR tt_content.header != '')
} }
pages { pages {
additionalWhereClause = pages.doktype NOT IN (3, 199, 6, 254, 255) additionalWhereClause = pages.doktype NOT IN (3, 199, 6, 254, 255)
abstractFields = abstract, description, bodytext abstractFields = abstract, description, bodytext
contentFields = header, bodytext
} }
} }
} }

View file

@ -20,6 +20,7 @@ plugin {
indexer = Codappix\SearchCore\Domain\Index\TcaIndexer\PagesIndexer indexer = Codappix\SearchCore\Domain\Index\TcaIndexer\PagesIndexer
additionalWhereClause = {$plugin.tx_searchcore.settings.indexing.pages.additionalWhereClause} additionalWhereClause = {$plugin.tx_searchcore.settings.indexing.pages.additionalWhereClause}
abstractFields = {$plugin.tx_searchcore.settings.indexing.pages.abstractFields} abstractFields = {$plugin.tx_searchcore.settings.indexing.pages.abstractFields}
contentFields = {$plugin.tx_searchcore.settings.indexing.pages.contentFields}
} }
} }

View file

@ -0,0 +1,14 @@
Changelog
=========
.. toctree::
:maxdepth: 1
:glob:
changelog/20180424-149-extract-relation-resolver-to-data-processing
changelog/20180410-148-keep-sys_language_uid
changelog/20180315-134-make-conent-fields-configurable
changelog/20180309-25-provide-sys-language-uid
changelog/20180308-131-respect-page-cache-clear
changelog/20180308-introduce-php70-type-hints
changelog/20180306-120-facet-configuration

View file

@ -0,0 +1,40 @@
Breaking Change 120 "Pass facets configuration to Elasticsearch"
================================================================
In order to allow arbitrary facet configuration, we do not process the facet configuration anymore.
Instead integrators are able to configure facets for search service "as is". We just pipe the
configuration through.
Therefore the following, which worked before, does not work anymore:
.. code-block:: typoscript
:linenos:
:emphasize-lines: 4
plugin.tx_searchcore.settings.search {
facets {
category {
field = categories
}
}
}
Instead you have to provide the full configuration yourself:
.. code-block:: typoscript
:linenos:
:emphasize-lines: 4,6
plugin.tx_searchcore.settings.search {
facets {
category {
terms {
field = categories
}
}
}
}
You need to add line 4 and 6, the additional level ``terms`` for Elasticsearch.
See :issue:`120`.

View file

@ -0,0 +1,14 @@
Feature 131 "Pages do not get indexed if content has changed"
=============================================================
Previously we only used DataHandler hooks triggered when processing records. This way we did not
index a page when content has changed.
We now also use cache clear hooks of DataHandler to index pages whenever their cache get cleared.
This way we also index a page if an integrator configured to clear further pages if content was
changed.
Still there are limitations. We do not get informed for pages which got cleared due to attached
caches via TypoScript.
See :issue:`131`.

View file

@ -0,0 +1,12 @@
Breaking Change "Introduce PHP 7.0 TypeHints"
=============================================
As PHP evolved, we now migrate the whole code base to use PHP 7.0 type hints.
We do not use PHP 7.1 Type Hints, as some customers still need PHP 7.0 support.
Also we added missing methods to interfaces, that were already used in code.
As this leads to changed method signatures, most custom implementations of interfaces, or overwrites
of existing methods are broken.
To fix, just update the signatures as pointed out by PHP while running the code.

View file

@ -0,0 +1,16 @@
Feature 25 "Respect multiple languages" - Provide sys_language_uid
==================================================================
Previously we did not fetch ``sys_language_uid`` field from database. This prevented everyone from
working with multiple languages.
By not removing the field it gets indexed and provides a very basic way of implementing multiple
languages.
At least it's now possible to filter search results by current language for now. Still the records
are not "valid" as we do not add overlays for now.
This is a first step into full multi language support.
Martin Hummer already has a basic proof of concept, based on :ref:`concepts_dataprocessing` working,
depending on ``sys_language_uid``.
See :issue:`25`.

View file

@ -0,0 +1,13 @@
Feature 134 "Enable indexing of tt_content records of CType Header"
===================================================================
Before, only ``bodytext`` was used to generate content while indexing pages.
As there are content elements like ``header`` where this field is empty, but content is still
available, it's now possible to configure the fields.
This makes it also possible to configure further custom content elements with new columns.
A new TypoScript option is now available, and ``header`` is added by default, see
:ref:`contentFields`.
See :issue:`134`.

View file

@ -0,0 +1,10 @@
Feature 148 "Cast sys_language_uid to int"
==========================================
While resolving relations the configured language uid field, fetched from TCA, will
be casted to integer and returned immediately.
This change prevents the bug mentioned in :issue:`148`, where `0` is casted to an
empty string, which makes filtering hard.
See :issue:`148`.

View file

@ -0,0 +1,24 @@
Breaking Change 149 "Extract RelationResolver to a new DataProcessor"
=====================================================================
The resolving of relation, based on TCA, is no longer done by the indexer. Instead we
now provide a DataProcessor to solve this job.
As this makes it necessary to configure the DataProcessor, this is a breaking change.
Before the resolving was done out of the box.
So why did we change that? The resolving of relations was already implemented before
we added the data processors. As the concept of data processors is far more flexible,
we try to migrate hard coupled components step by step. The benefit of this change is
basically that you can now configure the resolving of relations and far more
important, the order of execution.
Now it's possible to first copy some fields, e.g. ``starttime`` and ``endtime`` to
further fields and to resolve relations afterwards. As the copied fields are not
configured in TCA, they will be skipped. This way an integrator can keep certain
information as they are.
Also the processor is now configured as all other processors. You can now optionally
configure fields to not process.
See :issue:`149` and :issue:`147`.

View file

@ -3,38 +3,47 @@
Concepts Concepts
======== ========
The extension is built with the following concepts in mind. The main concept is to provide a foundation where other developers can profit from, to provide
integrations into search services like Elasticsearch, Algolia, ... .
Our code contains the following concepts which should be understand:
.. _concepts_connections: .. _concepts_connections:
Connections Connections
----------- -----------
It should be possible to use different search services like elasticsearch and solr out of the box. Different search services can provide integrations. ``search_core`` only provides abstractions and
If a service is not contained, it should be possible to implement the necessary part by implementing interfaces. The main purpose is to provide a stable API between TYPO3 and concrete connection.
the necessary interfaces and configuring the extension to use the new connection.
Also it should be possible to use multiple connections at once. This way multiple search services For information about implementing a new connection, take a look at :ref:`development_connection`.
can be used in the same installation.
Currently only :ref:`Elasticsearch` is provided.
.. _concepts_indexing: .. _concepts_indexing:
Indexing Indexing
-------- --------
The indexing is done by one of the available indexer. For each identifier it's possible to define Indexing is the process of collecting and preparing data, before sending it to a Connection.
the indexer to use. Also it's possible to write custom indexer to use. The indexing is done by one of the available indexer. Indexer are identified by a key, as configured
in TypoScript.
Currently only the :ref:`TcaIndexer` is provided. Currently :ref:`TcaIndexer` and :ref:`PagesIndexer` are provided.
.. _concepts_indexing_dataprocessing: For information about implementing a new indexer, take a look at :ref:`development_indexer`.
.. _concepts_dataprocessing:
DataProcessing DataProcessing
^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^
Before data is transfered to search service, it can be processed by "DataProcessors" like already Before data is transfered to search service, it can be processed by "DataProcessors" like already
known by :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing` of :ref:`t3tsref:cobj-fluidtemplate`. known by :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing` of :ref:`t3tsref:cobj-fluidtemplate`.
The same is true for retrieved search results. They can be processed again by "DataProcessors" to
prepare data for display in Templates or further usage.
Configuration is done through TypoScript, see :ref:`dataProcessing`. This should keep indexers simple and move logic to DataProcessors. This makes most parts highly
flexible as integrators are able to configure DataProcessors and change their order.
Configuration is done through TypoScript, see :ref:`dataprocessors`.
For information about implementing a new DataProcessor, take a look at :ref:`development_dataprocessor`.

View file

@ -51,7 +51,7 @@ master_doc = 'index'
# General information about the project. # General information about the project.
project = u'TYPO3 Extension search_core' project = u'TYPO3 Extension search_core'
copyright = u'2016, Daniel Siepmann' copyright = u'2016 - 2018, Daniel Siepmann'
author = u'Daniel Siepmann' author = u'Daniel Siepmann'
# The version info for the project you're documenting, acts as replacement for # The version info for the project you're documenting, acts as replacement for
@ -59,10 +59,9 @@ author = u'Daniel Siepmann'
# built documents. # built documents.
# #
# The short X.Y version. # The short X.Y version.
version = u'1.0.0' version = u'0.0.4'
# The full version, including alpha/beta/rc tags. # The full version, including alpha/beta/rc tags.
release = u'1.0.0' release = u'0.0.4'
# The language for content autogenerated by Sphinx. Refer to documentation # The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages. # for a list of supported languages.
# #
@ -133,7 +132,7 @@ html_theme_options = {
# The name for this set of Sphinx documents. # The name for this set of Sphinx documents.
# "<project> v<release> documentation" by default. # "<project> v<release> documentation" by default.
#html_title = u'TYPO3 Extension search_core v1.0.0' #html_title = u'TYPO3 Extension search_core v0.0.4'
# A shorter title for the navigation bar. Default is the same as html_title. # A shorter title for the navigation bar. Default is the same as html_title.
#html_short_title = None #html_short_title = None
@ -150,7 +149,7 @@ html_theme_options = {
# Add any paths that contain custom static files (such as style sheets) here, # Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files, # relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css". # so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static'] # html_static_path = ['_static']
# Add any extra paths that contain custom files (such as robots.txt or # Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied # .htaccess) here, relative to this directory. These files are copied

View file

@ -5,9 +5,12 @@
Configuration Configuration
============= =============
The extension offers the following configuration options through TypoScript. If you overwrite them Installation wide configuration is handled inside of the extension manager. Just check out the
through `setup` make sure to keep them in the `module` area as they will be accessed from backend options there, they all have labels.
mode of TYPO3. Do so by placing the following line at the end::
Everything else is configured through TypoScript. If you overwrite them through `setup` make sure to
keep them in the `module` area as they will be accessed from backend mode of TYPO3 for indexing. Do
so by placing the following line at the end::
module.tx_searchcore < plugin.tx_searchcore module.tx_searchcore < plugin.tx_searchcore
@ -23,12 +26,10 @@ Here is the example default configuration that's provided through static include
.. literalinclude:: ../../Configuration/TypoScript/constants.txt .. literalinclude:: ../../Configuration/TypoScript/constants.txt
:language: typoscript :language: typoscript
:linenos:
:caption: Static TypoScript Constants :caption: Static TypoScript Constants
.. literalinclude:: ../../Configuration/TypoScript/setup.txt .. literalinclude:: ../../Configuration/TypoScript/setup.txt
:language: typoscript :language: typoscript
:linenos:
:caption: Static TypoScript Setup :caption: Static TypoScript Setup
.. _configuration_options: .. _configuration_options:

View file

@ -4,7 +4,7 @@ Connections
=========== ===========
Holds settings regarding the different possible connections for search services like Elasticsearch Holds settings regarding the different possible connections for search services like Elasticsearch
or Solr. or Algolia.
Configured as:: Configured as::
@ -22,15 +22,13 @@ Configured as::
Where ``connectionName`` is one of the available :ref:`connections`. Where ``connectionName`` is one of the available :ref:`connections`.
The following settings are available. For each setting its documented which connection consumes it. The following settings are available.
.. _host: .. _host:
``host`` ``host``
-------- --------
Used by: :ref:`Elasticsearch`.
The host, e.g. ``localhost`` or an IP where the search service is reachable from TYPO3 The host, e.g. ``localhost`` or an IP where the search service is reachable from TYPO3
installation. installation.
@ -43,8 +41,6 @@ Example::
``port`` ``port``
-------- --------
Used by: :ref:`Elasticsearch`.
The port where search service is reachable. E.g. default ``9200`` for Elasticsearch. The port where search service is reachable. E.g. default ``9200`` for Elasticsearch.
Example:: Example::
@ -63,6 +59,3 @@ The index where the documents are being indexed to. E.g. default ``typo3content`
Example:: Example::
plugin.tx_searchcore.settings.connections.elasticsearch.index = typo3content plugin.tx_searchcore.settings.connections.elasticsearch.index = typo3content

View file

@ -0,0 +1,32 @@
``Codappix\SearchCore\DataProcessing\ContentObjectDataProcessorAdapterProcessor``
=================================================================================
Will execute an existing TYPO3 data processor.
Possible Options:
``_dataProcessor``
Necessary, defined which data processor to apply. Provide the same as you would to call the
processor.
``_table``
Defines the "current" table as used by some processors, e.g.
``TYPO3\CMS\Frontend\DataProcessing\FilesProcessor``.
All further options are passed to the configured data processor. Therefore they are documented at
each data processor.
Example::
plugin.tx_searchcore.settings.searching.dataProcessing {
1 = Codappix\SearchCore\DataProcessing\ContentObjectDataProcessorAdapterProcessor
1 {
_table = pages
_dataProcessor = TYPO3\CMS\Frontend\DataProcessing\FilesProcessor
references.fieldName = media
as = images
}
}
The above example will create a new field ``images`` with resolved FAL relations from ``media``
field.

View file

@ -8,6 +8,10 @@ Possible Options:
``to`` ``to``
Defines the field to copy the values into. All values not false will be copied at the moment. Defines the field to copy the values into. All values not false will be copied at the moment.
``from``
Optional, defines the field to copy, can only be one field.
If empty, all existing fields will be copied.
Example:: Example::
plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing { plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing {
@ -17,7 +21,8 @@ Example::
} }
2 = Codappix\SearchCore\DataProcessing\CopyToProcessor 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor
2 { 2 {
to = spellcheck from = uid
to = backup_uid
} }
} }

View file

@ -1,7 +1,7 @@
``Codappix\SearchCore\DataProcessing\RemoveProcessor`` ``Codappix\SearchCore\DataProcessing\RemoveProcessor``
====================================================== ======================================================
Will remove fields from record, e.g. if you do not want to sent them to elasticsearch at all. Will remove fields from record.
Possible Options: Possible Options:

View file

@ -0,0 +1,34 @@
``Codappix\SearchCore\DataProcessing\TcaRelationResolvingProcessor``
====================================================================
Will resolve relations through TCA for record.
The result will be the same as in list view of TYPO3 Backend. E.g. Check boxes will be
resolved to their label, dates will be resolved to human readable representation and
relations will be resolved to their configured labels.
Combine with CopyToProcessor or exclude certain fields to keep original value for
further processing.
Mandatory Options:
``_table``
The TCA table as found on top level of ``$GLOBALS['TCA']``.
This will auto filled for indexing through the provided indexers. Still you can
apply processors on results, where no information about the table exists anymore.
Possible Options:
``excludeFields``
Comma separated list of fields to not resolve relations for.
Example::
plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing {
1 = Codappix\SearchCore\DataProcessing\TcaRelationResolvingProcessor
1 {
_table = tt_content
excludeFields = starttime, endtime
}
}

View file

@ -40,11 +40,7 @@ Contains a comma separated list of page uids. Spaces are trimmed.
Example:: Example::
plugin.tx_searchcore.settings.indexing.<identifier>.rootLineBlacklist = 3, 10, 100 plugin.tx_searchcore.settings.indexing.pages.rootLineBlacklist = 3, 10, 100
Also it's possible to define some behaviour for the different document types. In context of TYPO3
tables are used as document types 1:1. It's possible to configure different tables. The following
options are available:
.. _additionalWhereClause: .. _additionalWhereClause:
@ -55,16 +51,16 @@ Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`.
Add additional SQL to where clauses to determine indexable records from the table. This way you Add additional SQL to where clauses to determine indexable records from the table. This way you
can exclude specific records like ``tt_content`` records with specific ``CType`` values or can exclude specific records like ``tt_content`` records with specific ``CType`` values or
something else. E.g. you can add a new field to the table to exclude records from indexing. something else.
Example:: Example::
plugin.tx_searchcore.settings.indexing.<identifier>.additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu') plugin.tx_searchcore.settings.indexing.tt_content.additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu')
.. attention:: .. attention::
Make sure to prefix all fields with the corresponding table name. The selection from Make sure to prefix all fields with the corresponding table name. The selection from
database will contain joins and can lead to SQL errors if a field exists in multiple tables. database might contain joins and can lead to SQL errors if a field exists in multiple tables.
.. _abstractFields: .. _abstractFields:
@ -73,6 +69,10 @@ abstractFields
Used by: :ref:`PagesIndexer`. Used by: :ref:`PagesIndexer`.
.. note::
Will be migrated to :ref:`dataprocessors` in the future.
Define which field should be used to provide the auto generated field "search_abstract". Define which field should be used to provide the auto generated field "search_abstract".
The fields have to exist in the record to be indexed. Therefore fields like ``content`` are also The fields have to exist in the record to be indexed. Therefore fields like ``content`` are also
possible. possible.
@ -80,21 +80,38 @@ possible.
Example:: Example::
# As last fallback we use the content of the page # As last fallback we use the content of the page
plugin.tx_searchcore.settings.indexing.<identifier>.abstractFields := addToList(content) plugin.tx_searchcore.settings.indexing.pages.abstractFields := addToList(content)
Default:: Default::
abstract, description, bodytext abstract, description, bodytext
.. _contentFields:
contentFields
-------------
Used by: :ref:`PagesIndexer`.
Define which fields should be used to provide the auto generated field "content".
Example::
plugin.tx_searchcore.settings.indexing.pages.contentFields := addToList(table_caption)
Default::
header, bodytext
.. _mapping: .. _mapping:
mapping mapping
------- -------
Used by: Elasticsearch connection while indexing. Used by: :ref:`connection_elasticsearch` connection while indexing.
Define mapping for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/mapping.html Define mapping for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/mapping.html
You are able to define the mapping for each property / columns. You are able to define the mapping for each property / column.
Example:: Example::
@ -112,7 +129,7 @@ makes building a facet possible.
index index
----- -----
Used by: Elasticsearch connection while indexing. Used by: :ref:`connection_elasticsearch` connection while indexing.
Define index for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/indices-create-index.html Define index for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/indices-create-index.html
@ -141,67 +158,12 @@ Example::
``char_filter`` and ``filter`` are a comma separated list of options. ``char_filter`` and ``filter`` are a comma separated list of options.
.. _dataProcessing: .. _indexing_dataProcessing:
dataProcessing dataProcessing
-------------- --------------
Used by: All connections while indexing. Used by: All connections while indexing, due to implementation inside ``AbstractIndexer``.
Configure modifications on each document before sending it to the configured connection. Same as Configure modifications on each document before sending it to the configured connection.
provided by TYPO3 for :ref:`t3tsref:cobj-fluidtemplate` through For full documentation check out :ref:`dataprocessors`.
:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`.
All processors are applied in configured order. Allowing to work with already processed data.
Example::
plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing {
1 = Codappix\SearchCore\DataProcessing\CopyToProcessor
1 {
to = search_spellcheck
}
2 = Codappix\SearchCore\DataProcessing\CopyToProcessor
2 {
to = search_all
}
}
The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards
all fields, including ``search_spellcheck`` will be copied to ``search_all``.
E.g. used to index all information into a field for :ref:`spellchecking` or searching with
different :ref:`mapping`.
The following Processor are available:
.. toctree::
:maxdepth: 1
:glob:
dataProcessing/CopyToProcessor
dataProcessing/RemoveProcessor
dataProcessing/GeoPointProcessor
The following Processor are planned:
``Codappix\SearchCore\DataProcessing\ReplaceProcessor``
Will execute a search and replace on configured fields.
``Codappix\SearchCore\DataProcessing\RootLevelProcessor``
Will attach the root level to the record.
``Codappix\SearchCore\DataProcessing\ChannelProcessor``
Will add a configurable channel to the record, e.g. if you have different areas in your
website like "products" and "infos".
``Codappix\SearchCore\DataProcessing\RelationResolverProcessor``
Resolves all relations using the TCA.
Of course you are able to provide further processors. Just implement
``Codappix\SearchCore\DataProcessing\ProcessorInterface`` and use the FQCN (=Fully qualified
class name) as done in the examples above.
By implementing also the same interface as necessary for TYPO3
:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`, you are able to reuse the same code
also for Fluid to prepare the same record fetched from DB for your fluid.

View file

@ -8,45 +8,50 @@ Searching
size size
---- ----
Used by: Elasticsearch connection while building search query.
Defined how many search results should be fetched to be available in search result. Defined how many search results should be fetched to be available in search result.
Example:: Example::
plugin.tx_searchcore.settings.searching.size = 50 plugin.tx_searchcore.settings.searching.size = 50
Default if not configured is 10. Default is ``10``.
.. _facets: .. _facets:
facets facets
------ ------
Used by: Elasticsearch connection while building search query.
Define aggregations for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-aggregations-bucket-terms-aggregation.html Define aggregations for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-aggregations-bucket-terms-aggregation.html
Currently only the term facet is provided.
Example:: Example::
plugin.tx_searchcore.settings.searching.facets { category {
contentTypes { terms {
field = CType field = categories
} }
} }
The above example will provide a facet with options for all found ``CType`` results together month {
with a count. date_histogram {
field = released
interval = month
format = Y-MM-01
order {
_time = desc
}
}
}
The above example will provide a facet with options for all found ``categories`` results together
with a count. Also a facet for ``released`` will be provided.
.. _filter: .. _filter:
filter filter
------ ------
Used by: While building search request. Define filter that should be set for all search requests.
Define filter that should be set for all requests.
Example:: Example::
@ -54,6 +59,9 @@ Example::
property = value property = value
} }
Also see :ref:`mapping.filter` to map incoming request information, e.g. from a ``select``, to build
more complex filters.
For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``keyword``. For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``keyword``.
.. _minimumShouldMatch: .. _minimumShouldMatch:
@ -61,9 +69,8 @@ For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``key
minimumShouldMatch minimumShouldMatch
------------------ ------------------
Used by: Elasticsearch connection while building search query. Define the minimum match for Elasticsearch, have a look at the official docs:
https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-minimum-should-match.html
Define the minimum match for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-minimum-should-match.html
Example:: Example::
@ -74,8 +81,6 @@ Example::
boost boost
----- -----
Used by: Elasticsearch connection while building search query.
Define fields that should boost the score for results. Define fields that should boost the score for results.
Example:: Example::
@ -93,10 +98,9 @@ https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_boosting_query_clauses.
fieldValueFactor fieldValueFactor
---------------- ----------------
Used by: Elasticsearch connection while building search query. Define a field to use as a factor for scoring. The configuration is passed through to Elasticsearch
``field_value_factor``, see:
Define a field to use as a factor for scoring. The configuration is passed through to elastic https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor
search ``field_value_factor``, see: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor
Example:: Example::
@ -119,55 +123,84 @@ E.g. you submit a filter in form of:
.. code-block:: html .. code-block:: html
<f:form.textfield property="filter.distance.location.lat" value="51.168098" /> <f:comment>
<f:form.textfield property="filter.distance.location.lon" value="6.381384" /> Due to TYPO3 7.x fluid limitations, we build this input ourself.
<f:form.textfield property="filter.distance.distance" value="100km" /> No longer necessary in 8 and above
</f:comment>
<select name="tx_searchcore_search[searchRequest][filter][month][from]" class="_control" >
<option value="">Month</option>
<f:for each="{searchResult.facets.month.options}" as="month">
<f:if condition="{month.count}">
<option
value="{month.displayName -> f:format.date(format: 'Y-m')}"
{f:if(condition: '{searchRequest.filter.month.from} == {month.displayName -> f:format.date(format: \'Y-m\')}', then: 'selected="true"')}
>{month.displayName -> f:format.date(format: '%B %Y')}</option>
</f:if>
</f:for>
</select>
<select name="tx_searchcore_search[searchRequest][filter][month][to]" class="_control" >
<option value="">Month</option>
<f:for each="{searchResult.facets.month.options}" as="month">
<f:if condition="{month.count}">
<option
value="{month.displayName -> f:format.date(format: 'Y-m')}"
{f:if(condition: '{searchRequest.filter.month.from} == {month.displayName -> f:format.date(format: \'Y-m\')}', then: 'selected="true"')}
>{month.displayName -> f:format.date(format: '%B %Y')}</option>
</f:if>
</f:for>
</select>
This will create a ``distance`` filter with subproperties. To make this filter actually work, you This will create a ``month`` filter with sub properties. To make this filter actually work, you
can add the following TypoScript, which will be added to the filter:: can add the following TypoScript, which will be added to the filter::
mapping { mapping {
filter { filter {
distance { month {
field = geo_distance type = range
field = released
raw {
format = yyyy-MM
}
fields { fields {
distance = distance gte = from
location = location lte = to
} }
} }
} }
} }
``fields`` has a special meaning here. This will actually map the properties of the filter to fields ``fields`` has a special meaning here. This will actually map the properties of the filter to fields
in elasticsearch. In above example they do match, but you can also use different names in your form. in Elasticsearch. On the left hand side is the Elasticsearch field name, on the right side the one
On the left hand side is the elasticsearch field name, on the right side the one submitted as a submitted as a filter.
filter.
The ``field``, in above example ``geo_distance``, will be used as the elasticsearch field for The ``field``, in above example ``released``, will be used as the Elasticsearch field for
filtering. This way you can use arbitrary filter names and map them to existing elasticsearch fields. filtering. This way you can use arbitrary filter names and map them to existing Elasticsearch fields.
Everything that is configured inside ``raw`` is passed, as is, to search service, e.g.
Elasticsearch.
.. _fields: .. _fields:
fields fields
------ ------
Defines the fields to fetch and search from elasticsearch. With the following sub keys: Defines the fields to fetch and search from Elasticsearch. With the following sub keys:
``query`` defines the fields to search in. Default is ``_all`` from 5.x times of elasticsearch. ``query`` defines the fields to search in. Configure a comma separated list of fields to search in.
Configure a comma separated list of fields to search in. This is necessary if you have configured This is necessary if you have configured special mapping for some fields, or just want to search
special mapping for some fields, or just want to search some fields. some fields. The following is an example configuration::
The most hits get ranked highest. The following is an example configuration::
fields { fields {
query = _all, city query = _all, city
} }
The following sub properties configure the fields to fetch from elasticsearch: The following sub properties configure the fields to fetch from Elasticsearch:
First ``stored_fields`` which is a list of comma separated fields which actually exist and will be First ``stored_fields`` which is a list of comma separated fields which actually exist and will be
added. Typically you will use ``_source`` to fetch the whole indexed fields. added. Typically you will use ``_source`` to fetch the whole indexed fields.
Second is ``script_fields``, which allow you to configure scripted fields for elasticsearch. Second is ``script_fields``, which allow you to configure scripted fields for Elasticsearch.
An example might look like the following:: An example might look like the following::
fields { fields {
@ -190,7 +223,7 @@ In above example we add a single ``script_field`` called ``distance``. We add a
field should be added. The condition will be parsed as Fluidtemplate and is casted to bool via PHP. field should be added. The condition will be parsed as Fluidtemplate and is casted to bool via PHP.
If the condition is true, or no ``condition`` exists, the ``script_field`` will be added to the If the condition is true, or no ``condition`` exists, the ``script_field`` will be added to the
query. The ``condition`` will be removed and everything else is submitted one to one to query. The ``condition`` will be removed and everything else is submitted one to one to
elasticsearch, except each property is run through Fluidtemplate, to allow you to use information Elasticsearch, except each property is run through Fluidtemplate, to allow you to use information
from search request, e.g. to insert latitude and longitude from a filter, like in the above example. from search request, e.g. to insert latitude and longitude from a filter, like in the above example.
.. _sort: .. _sort:
@ -215,4 +248,14 @@ Example::
mode = filter mode = filter
} }
Only ``filter`` is allowed as value. Will submit an empty query to switch to filter mode. Only ``filter`` is allowed as value, as ``search`` is default behaviour. Using ``filter`` will
trigger a search to provide data while visiting the page, possible :ref:`filter` allow you to build
pages like "News".
.. _searching_dataprocessing:
dataProcessing
--------------
Configure modifications on each document before returning search result.
For full documentation check out :ref:`dataprocessors`.

View file

@ -5,28 +5,25 @@ Connections
See Concept of :ref:`concepts_connections` for further background information. See Concept of :ref:`concepts_connections` for further background information.
The extension provides the following connections out of the box: For information about implementing a new connection, take a look at :ref:`development_connection`.
.. _Elasticsearch: The following connections were developed, or are in development, for ``search_core``:
.. _connection_elasticsearch:
Elasticsearch Elasticsearch
------------- -------------
Integrates `elastic Elasticsearch`_ using `elastica`_ into TYPO3. Support for `Elasticsearch`_ is provided out of the box by `search_core` at the moment.
Provides basic support like indexing without mappings and full text search at the moment. .. _Elasticsearch: https://www.elastic.co/products/elasticsearch
The connection is configurable through the following options: .. _connection_algolia:
* :ref:`host` Algolia
-------
* :ref:`port` `search_algolia`_ will integrate `Algolia`_ and is currently under development by Martin Hummer.
* :ref:`index` .. _search_algolia: https://github.com/martinhummer/search_algolia/
.. _Algolia: https://www.algolia.com/
* :ref:`mapping`
* :ref:`facets`
.. _elastic Elasticsearch: https://www.elastic.co/products/elasticsearch
.. _elastica: http://elastica.io/

View file

@ -0,0 +1,84 @@
.. _dataprocessors:
DataProcessors
==============
See Concept of :ref:`concepts_dataprocessing` for further background information.
For information about implementing a new DataProcessor, take a look at
:ref:`development_dataprocessor`.
Same as provided by TYPO3 for :ref:`t3tsref:cobj-fluidtemplate` through
:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`.
.. _dataprocessors_usage:
Usage
-----
All processors are applied in configured order. Allowing to work with already processed data.
They can be applied during indexing and for search results.
Example for indexing::
plugin.tx_searchcore.settings.indexing.pages.dataProcessing {
1 = Codappix\SearchCore\DataProcessing\CopyToProcessor
1 {
to = search_spellcheck
}
2 = Codappix\SearchCore\DataProcessing\CopyToProcessor
2 {
to = search_all
}
}
The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards
all fields, including ``search_spellcheck`` will be copied to ``search_all``.
Example for search results::
plugin.tx_searchcore.settings.searching.dataProcessing {
1 = Codappix\SearchCore\DataProcessing\CopyToProcessor
1 {
to = search_spellcheck
}
2 = Codappix\SearchCore\DataProcessing\CopyToProcessor
2 {
to = search_all
}
}
The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards
all fields, including ``search_spellcheck`` will be copied to ``search_all``.
.. _dataprocessors_availableDataProcessors:
Available DataProcessors
------------------------
.. toctree::
:maxdepth: 1
:glob:
/configuration/dataProcessing/ContentObjectDataProcessorAdapterProcessor
/configuration/dataProcessing/CopyToProcessor
/configuration/dataProcessing/GeoPointProcessor
/configuration/dataProcessing/RemoveProcessor
/configuration/dataProcessing/TcaRelationResolvingProcessor
.. _dataprocessors_plannedDataProcessors:
Planned DataProcessors
----------------------
``Codappix\SearchCore\DataProcessing\ReplaceProcessor``
Will execute a search and replace on configured fields.
``Codappix\SearchCore\DataProcessing\RootLevelProcessor``
Will attach the root level to the record.
``Codappix\SearchCore\DataProcessing\ChannelProcessor``
Will add a configurable channel to the record, e.g. if you have different areas in your
website like "products" and "infos".

View file

@ -1,71 +1,16 @@
.. highlight:: bash .. highlight:: bash
.. _contribution:
Contribution
============
Everyone is welcome to contribute, whether it's code, issues, feature requests or any other kind.
Below is a documentation what to respect during contributions.
.. _contribution_setup:
Setup
-----
To start contributions regarding code, make sure your environment matches the following
requirements:
* composer is executable
* PHP on CLI is executable
* MySQL is up and running with user *dev* and password *dev* on *127.0.0.1* or to overwrite the
environment variables, see :file:`Makefile`.
And MySQL is not set to strict mode as TYPO3 doesn't support strict mode, see
https://review.typo3.org/#/c/26725/3/INSTALL.md.
* Elasticsearch is installed and up and running on *localhost:9200*.
Then setup your system::
git clone git@github.com:DanielSiepmann/search_core.git \
&& cd search_core \
&& export typo3DatabaseName="searchcoretest76" \
&& export TYPO3_VERSION="~7.6" \
&& make install \
&& make unitTests \
&& make functionalTests
If all tests are okay, start your work.
If you are working with multiple TYPO3 versions make sure to export `typo3DatabaseName` and
`TYPO3_VERSION` in your environment like::
export typo3DatabaseName="searchcoretest62"
export TYPO3_VERSION="~6.2"
Also run the install command for each version before running any tests. Only this will make sure you
are testing against the actual TYPO3 Version and database scheme.
.. _contribution_development:
Development Development
----------- ===========
All changes are introduced through pull requests at `Github`_ and should contain the following: There are some ways we will cover here. One is how you can develop own parts like Indexer,
DataProcessor and Connection. The other is how to contribute.
* Adjusted tests if tests existed before. Otherwise they will break on `travis-ci`_. .. toctree::
:maxdepth: 1
:glob:
* New tests whenever possible and useful. development/indexer
development/dataProcessor
* Code has to follow `PSR-2`_. development/connection
development/contribution
* Adjusted documentation.
* Make sure to follow the documented :ref:`concepts`.
.. _Github: https://github.com/DanielSiepmann/search_core
.. _travis-ci: https://travis-ci.org/
.. _PSR-2: http://www.php-fig.org/psr/psr-2/

View file

@ -0,0 +1,11 @@
.. _development_connection:
Develop a new Connection
========================
Make sure you understood :ref:`concepts_connections`.
Each Connection has to be a single class which implements
``Codappix\SearchCore\Connection\ConnectionInterface``.
Dependency Injection is working for custom Connections.

View file

@ -0,0 +1,69 @@
.. _contribution:
Contribution
============
Everyone is welcome to contribute, whether it's code, issues, feature requests or any other kind.
Below is a documentation what to respect during contributions.
.. _contribution_setup:
Setup
-----
To start contributions regarding code, make sure your environment matches the following
requirements:
* composer is executable
* PHP on CLI is executable
* MySQL is up and running with user *dev* and password *dev* on *127.0.0.1* or to overwrite the
environment variables, see :file:`Makefile`.
And MySQL is not set to strict mode as TYPO3 doesn't support strict mode, see
https://review.typo3.org/#/c/26725/3/INSTALL.md.
* Elasticsearch is installed and up and running on *localhost:9200*.
Then setup your system::
git clone git@github.com:codappix/search_core.git \
&& cd search_core \
&& export typo3DatabaseName="searchcoretest87" \
&& export TYPO3_VERSION="~8.7" \
&& make install \
&& make unitTests \
&& make functionalTests
If all tests are okay, start your work.
If you are working with multiple TYPO3 versions make sure to export `typo3DatabaseName` and
`TYPO3_VERSION` in your environment like::
export typo3DatabaseName="searchcoretest76"
export TYPO3_VERSION="~7.6"
Also run the install command for each version before running any tests. Only this will make sure you
are testing against the actual TYPO3 Version and database scheme.
.. _contribution_development:
Development
-----------
All changes are introduced through pull requests at `Github`_ and should contain the following:
* Adjusted tests if tests existed before. Otherwise they will break on `travis-ci`_.
* New tests whenever possible and useful.
* Code has to follow `PSR-2`_.
* Adjusted documentation.
* Make sure to follow the documented :ref:`concepts`.
.. _Github: https://github.com/codappix/search_core
.. _travis-ci: https://travis-ci.org/
.. _PSR-2: http://www.php-fig.org/psr/psr-2/

View file

@ -0,0 +1,15 @@
.. _development_dataprocessor:
Develop a new DataProcessor
===========================
Make sure you understood :ref:`concepts_dataprocessing`.
Each DataProcessor has to be a single class which implements
``Codappix\SearchCore\DataProcessing\ProcessorInterface``.
Make sure you support both, Frontend and Backend, as processors can be called during searching and
indexing. Therefore do not rely on e.g. ``TSFE``, make sure dependencies are met and your code will
work in both environments.
Dependency Injection is working for custom DataProcessors.

View file

@ -0,0 +1,21 @@
.. _development_indexer:
Develop a new Indexer
=====================
Make sure you understood :ref:`concepts_indexing`.
Each indexer has to be a single class which implements
``Codappix\SearchCore\Domain\Index\IndexerInterface``.
The indexer should call the connection with all necessary information about the document(s) to
trigger indexing or deletion of whole index.
As this is the "indexer", deletion of single documents is directly processed by the connection.
``setIdentifier`` is called with the identifier of the current Indexer. This might be useful to
fetch configuration, related to the indexing, from
``Codappix\SearchCore\Configuration\ConfigurationContainerInterface``.
Dependency Injection is working for custom indexers, therefore you are able to inject the
``ConfigurationContainerInterface``.

View file

@ -3,38 +3,60 @@
Features Features
======== ========
The following features are currently provided: The following features are available:
.. _features_indexing: .. _features_indexing:
Indexing Indexing
-------- --------
Indexing data to Elasticsearch is provided. The extension delivers an indexer for TCA with zero Indexing of data is possible. We deliver an indexer for TCA with zero configuration needs. You can
configuration needs. Still it's possible to configure the indexer. also provide custom indexer for arbitrary data which is not indexable through TCA.
Also custom classes can be used as indexers. Also a finisher for TYPO3 Form-Extension is provided to integrate indexing after something was
update through the Form-Extension.
Furthermore a finisher for TYPO3 Form-Extension is provided to integrate indexing. Indexing is done through Hooks and CLI. We therefore provide commands to index and delete indexed
data.
.. _features_search: .. _features_search:
Searching Searching
--------- ---------
Currently all fields are searched for a single search input. .. note::
Currently only integrated for Elasticsearch with no abstraction.
If you need to implement your own search, please open an issue on Github and we will change the code
base.
Also multiple filter are supported. Filtering results by fields for string contents. Via TypoScript it's possible to configure the fields to query, minimum match and script fields.
Also multiple filter are supported, filtering results by fields.
Facets / aggregates are also possible. Therefore a mapping has to be defined in TypoScript for Facets / aggregates are also possible. Therefore a mapping has to be defined in TypoScript for
indexing, and the facets itself while searching. indexing, and the facets itself while searching.
.. _features_dataProcessing:
DataProcessing
--------------
DataProcessing, as known from ``FLUIDTEMPLATE``, is available while indexing and for search results.
Each record and result item can be processed by multiple processor to prepare data for indexing and
output.
See :ref:`concepts_dataprocessing` in :ref:`concepts` section.
.. _features_planned: .. _features_planned:
Planned Planned
--------- -------
The following features are currently planned and will be integrated: The following features are currently planned and will be integrated:
#. Pagination #. :issue:`25` Multi language.
Add a pagination to search results, to allow users to walk through all results. #. :issue:`94` Respect access rights while indexing relations.
#. :issue:`75` Configuration of index name (for Elasticsearch).
For a full list, check out our `open issues`_.
.. _open issues: https://github.com/Codappix/search_core/issues

View file

@ -1,7 +1,7 @@
.. include:: readme.rst .. include:: readme.rst
Table of Contents Table of Contents
================= -----------------
.. toctree:: .. toctree::
:maxdepth: 1 :maxdepth: 1
@ -14,4 +14,6 @@ Table of Contents
concepts concepts
connections connections
indexer indexer
dataprocessors
development development
changelog

View file

@ -5,6 +5,8 @@ Indexer
See Concept of :ref:`concepts_indexing` for further background information. See Concept of :ref:`concepts_indexing` for further background information.
For information about implementing a new indexer, take a look at :ref:`development_indexer`.
The extension provides the following indexer out of the box: The extension provides the following indexer out of the box:
.. _TcaIndexer: .. _TcaIndexer:
@ -15,23 +17,12 @@ TcaIndexer
Provides zero configuration TYPO3 integration by using the :ref:`t3tcaref:start`. You just can Provides zero configuration TYPO3 integration by using the :ref:`t3tcaref:start`. You just can
start indexing TYPO3. start indexing TYPO3.
The indexer will use the TCA to fetch all necessary information like relations. Currently the Just add the indexer for a TYPO3 table. The indexer will use the TCA to fetch all necessary
implementation is very basic. In future it will also provide mapping for :ref:`Elasticsearch` and information like relations.
further stuff.
The indexer is configurable through the following options: .. note::
* :ref:`rootLineBlacklist` Not all relations are resolved yet, see :issue:`17`.
* :ref:`additionalWhereClause`
* :ref:`abstractFields`
* :ref:`mapping`
* :ref:`index`
* :ref:`dataProcessing`
.. _PagesIndexer: .. _PagesIndexer:
@ -42,23 +33,7 @@ Provides zero configuration TYPO3 integration by using the :ref:`t3tcaref:start`
start indexing TYPO3. start indexing TYPO3.
The indexer will use the TCA to fetch all necessary information like relations. Currently the The indexer will use the TCA to fetch all necessary information like relations. Currently the
implementation is very basic. In future it will also provide mapping for :ref:`Elasticsearch` and implementation is very basic.
further stuff. Also all static content from each page will be concatenated into a single field to
improve search.
The indexer is configurable through the following options:
* :ref:`rootLineBlacklist`
* :ref:`additionalWhereClause`
* :ref:`abstractFields`
* :ref:`mapping`
* :ref:`index`
* :ref:`dataProcessing`
.. note:: .. note::

View file

@ -4,19 +4,41 @@
Installation Installation
============ ============
Composer
--------
The extension can be installed through composer:: The extension can be installed through composer::
composer require "leonmrni/search_core dev-master as 1.0.x-dev" composer require "codappix/search_core" "~0.0.4"
or by `downloading`_ and placing it inside the :file:`typo3conf/ext`-Folder of your installation. Note that you have to allow unstable packages:
In that case you need to install all dependencies yourself. Dependencies are:
.. code-block:: json
{
"minimum-stability": "dev",
"prefer-stable": true
}
Download
--------
You can also `download`_ the extension and placing it inside the :file:`typo3conf/ext`-Folder of
your installation. In that case you need to install all dependencies yourself. Dependencies are:
.. literalinclude:: ../../composer.json .. literalinclude:: ../../composer.json
:caption: Dependencies from composer.json :caption: Dependencies from composer.json
:lines: 19-21 :lines: 19-21
:dedent: 8 :dedent: 8
Setup
-----
Afterwards you need to enable the extension through the extension manager and include the static Afterwards you need to enable the extension through the extension manager and include the static
TypoScript setup. TypoScript setup.
.. _downloading: https://github.com/DanielSiepmann/search_core/archive/master.zip If you **don't** want to use the included Elasticsearch integration, you have to disable it in the
extension manager configuration of the extension by checking the checkbox.
It's currently enabled by default but will be moved into its own extension in the future.
.. _download: https://github.com/codappix/search_core/archive/develop.zip

View file

@ -1,5 +1,5 @@
TYPO3 Extension search_core's documentation! TYPO3 Extension search_core
============================================ ===========================
Introduction Introduction
============ ============
@ -8,24 +8,21 @@ What does it do?
---------------- ----------------
The goal of this extension is to provide search integrations into TYPO3 CMS. The extension will The goal of this extension is to provide search integrations into TYPO3 CMS. The extension will
abstract the concrete implementations to allow exchange of concrete backends like Elasticsearch or provide a convenient API to allow developers to provide concrete implementations of backends like
solr. Elasticsearch, Algolia or Solr.
The extension provides integration into TYPO3 like a frontend plugin for searches and hooks to The extension provides integration into TYPO3 like a frontend plugin for searches and hooks to
update search indexes on updates. Also a command line interface is provided for interactions like update search indexes on updates. Also a command line interface is provided for interactions like
reindexing. re-indexing.
Current state Current state
------------- -------------
This is still a very early beta version. More information can be taken from Github at This is still a very early beta version. More information can be taken from Github at
`current issues`_ and `current projects`_. `current issues`_.
We are also focusing on Code Quality and Testing through `travis ci`_, `scrutinizer`_ and `codacy`_. We are also focusing on Code Quality and Testing through `travis ci`_, ``phpcs``, ``phpunit`` and
``phpstan``.
.. _current issues: https://github.com/Codappix/search_core/issues .. _current issues: https://github.com/Codappix/search_core/issues
.. _current projects: https://github.com/Codappix/search_core/projects
.. _travis ci: https://travis-ci.org/Codappix/search_core .. _travis ci: https://travis-ci.org/Codappix/search_core
.. _scrutinizer: https://scrutinizer-ci.com/g/Codappix/search_core/inspections
.. _codacy: https://www.codacy.com/app/Codappix/search_core/dashboard

View file

@ -11,9 +11,10 @@ Manual indexing
You can trigger indexing from CLI:: You can trigger indexing from CLI::
./typo3/cli_dispatch.phpsh extbase index:index --identifier 'tt_content' ./typo3/cli_dispatch.phpsh extbase index:index --identifier 'pages'
./bin/typo3cms index:index --identifier 'pages'
This will index the table ``tt_content`` using the :ref:`TcaIndexer`. This will index the table ``pages`` using the :ref:`TcaIndexer`.
Only one index per call is available, to run multiple indexers, just make multiple calls. Only one index per call is available, to run multiple indexers, just make multiple calls.
The indexers have to be defined in TypoScript via :ref:`configuration_options_index`. The indexers have to be defined in TypoScript via :ref:`configuration_options_index`.
@ -25,9 +26,10 @@ Manual deletion
You can trigger deletion for a single index from CLI:: You can trigger deletion for a single index from CLI::
./typo3/cli_dispatch.phpsh extbase index:delete --identifier 'tt_content' ./typo3/cli_dispatch.phpsh extbase index:delete --identifier 'pages'
./bin/typo3cms index:delete --identifier 'pages'
This will delete the index for the table ``tt_content``. This will delete the index for the table ``pages``.
Only one delete per call is available, to run multiple deletions, just make multiple calls. Only one delete per call is available, to run multiple deletions, just make multiple calls.
@ -53,7 +55,6 @@ A form finisher is provided to integrate indexing into form extension.
Add form finisher to your available finishers and configure it like: Add form finisher to your available finishers and configure it like:
.. code-block:: yaml .. code-block:: yaml
:linenos:
- -
identifier: SearchCoreIndexer identifier: SearchCoreIndexer
@ -62,7 +63,7 @@ Add form finisher to your available finishers and configure it like:
indexIdentifier: 'fe_users' indexIdentifier: 'fe_users'
recordUid: '{FeUser.user.uid}' recordUid: '{FeUser.user.uid}'
All three options are necessary, where All three options are necessary, where:
``action`` ``action``
Is one of ``delete``, ``update`` or ``add``. Is one of ``delete``, ``update`` or ``add``.
@ -81,7 +82,7 @@ plugin. The plugin is named *Search Core*.
Please provide your own template, the extension will not deliver a useful template for now. Please provide your own template, the extension will not deliver a useful template for now.
The extbase mapping is used, this way you can create a form: The Extbase mapping is used, this way you can create a form:
.. code-block:: html .. code-block:: html
@ -95,11 +96,10 @@ The extbase mapping is used, this way you can create a form:
Filter Filter
"""""" """"""
Thanks to extbase mapping, filter are added to the form: Thanks to Extbase mapping, filter are added to the form:
.. code-block:: html .. code-block:: html
<!-- Case sensitive for fields of type keyword. -->
<f:form.textfield property="filter.exampleName" value="the value to match" /> <f:form.textfield property="filter.exampleName" value="the value to match" />
.. _usage_searching_facets: .. _usage_searching_facets:

View file

@ -5,16 +5,28 @@ TYPO3_WEB_DIR := $(current_dir).Build/web
TYPO3_PATH_ROOT := $(current_dir).Build/web TYPO3_PATH_ROOT := $(current_dir).Build/web
# Allow different versions on travis # Allow different versions on travis
TYPO3_VERSION ?= ~8.7 TYPO3_VERSION ?= ~8.7
typo3DatabaseName ?= "searchcore_test2" typo3DatabaseName ?= "searchcore_test"
typo3DatabaseUsername ?= "dev" typo3DatabaseUsername ?= "dev"
typo3DatabasePassword ?= "dev" typo3DatabasePassword ?= "dev"
typo3DatabaseHost ?= "127.0.0.1" typo3DatabaseHost ?= "127.0.0.1"
sourceOrDist=--prefer-dist
ifeq ($(TYPO3_VERSION),~7.6)
sourceOrDist=--prefer-source
endif
.PHONY: install .PHONY: install
install: clean install: clean
COMPOSER_PROCESS_TIMEOUT=1000 composer require -vv --dev --prefer-dist --ignore-platform-reqs typo3/cms="$(TYPO3_VERSION)" if [ $(TYPO3_VERSION) = ~7.6 ]; then \
patch composer.json Tests/InstallPatches/composer.json.patch; \
fi
COMPOSER_PROCESS_TIMEOUT=1000 composer require -vv --dev $(sourceOrDist) typo3/cms="$(TYPO3_VERSION)"
git checkout composer.json git checkout composer.json
cgl:
./.Build/bin/phpcs
functionalTests: functionalTests:
typo3DatabaseName=$(typo3DatabaseName) \ typo3DatabaseName=$(typo3DatabaseName) \
typo3DatabaseUsername=$(typo3DatabaseUsername) \ typo3DatabaseUsername=$(typo3DatabaseUsername) \
@ -29,16 +41,11 @@ unitTests:
.Build/bin/phpunit --colors --debug -v \ .Build/bin/phpunit --colors --debug -v \
-c Tests/Unit/UnitTests.xml -c Tests/Unit/UnitTests.xml
uploadCodeCoverage: uploadCodeCoverageToScrutinizer uploadCodeCoverageToCodacy uploadCodeCoverage: uploadCodeCoverageToScrutinizer
uploadCodeCoverageToScrutinizer: uploadCodeCoverageToScrutinizer:
wget https://scrutinizer-ci.com/ocular.phar && \ wget https://scrutinizer-ci.com/ocular.phar && \
php ocular.phar code-coverage:upload --format=php-clover .Build/report/functional/clover/coverage php ocular.phar code-coverage:upload --format=php-clover .Build/report/functional/clover/coverage
uploadCodeCoverageToCodacy:
composer require -vv --dev codacy/coverage && \
git checkout composer.json && \
php .Build/bin/codacycoverage clover .Build/report/functional/clover/coverage
clean: clean:
rm -rf .Build composer.lock rm -rf .Build composer.lock

View file

@ -66,4 +66,9 @@ abstract class AbstractFunctionalTestCase extends CoreTestCase
{ {
return ['EXT:search_core/Tests/Functional/Fixtures/BasicSetup.ts']; return ['EXT:search_core/Tests/Functional/Fixtures/BasicSetup.ts'];
} }
protected function isLegacyVersion() : bool
{
return \TYPO3\CMS\Core\Utility\VersionNumberUtility::convertVersionNumberToInteger(TYPO3_version) < 8000000;
}
} }

View file

@ -0,0 +1,9 @@
<?php
$filePath = '.Build/vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTestsBootstrap.php';
if (getenv('TYPO3_VERSION') === '~7.6') {
$filePath = '.Build/vendor/typo3/cms/typo3/sysext/core/Build/FunctionalTestsBootstrap.php';
}
require_once dirname(dirname(__DIR__)) . '/' . $filePath;

View file

@ -0,0 +1,78 @@
<?php
namespace Codappix\SearchCore\Tests\Functional\Connection\Elasticsearch;
/*
* Copyright (C) 2017 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
use Codappix\SearchCore\Domain\Index\IndexerFactory;
use Codappix\SearchCore\Domain\Model\SearchRequest;
use Codappix\SearchCore\Domain\Search\SearchService;
use TYPO3\CMS\Extbase\Object\ObjectManager;
class FacetTest extends AbstractFunctionalTestCase
{
protected function getTypoScriptFilesForFrontendRootPage()
{
return array_merge(
parent::getTypoScriptFilesForFrontendRootPage(),
['EXT:search_core/Tests/Functional/Fixtures/Searching/Facet.ts']
);
}
protected function getDataSets()
{
return array_merge(
parent::getDataSets(),
['Tests/Functional/Fixtures/Searching/Filter.xml']
);
}
/**
* @test
*/
public function itsPossibleToFetchFacetsForField()
{
\TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class)
->get(IndexerFactory::class)
->getIndexer('tt_content')
->indexAllDocuments()
;
$searchService = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class)
->get(SearchService::class);
$searchRequest = new SearchRequest();
$result = $searchService->search($searchRequest);
$this->assertSame(1, count($result->getFacets()), 'Did not receive the single defined facet.');
$facet = current($result->getFacets());
$this->assertSame('contentTypes', $facet->getName(), 'Name of facet was not as expected.');
$this->assertSame('CType', $facet->getField(), 'Field of facet was not expected.');
$options = $facet->getOptions();
$this->assertSame(2, count($options), 'Did not receive the expected number of possible options for facet.');
$option = $options['HTML'];
$this->assertSame('HTML', $option->getName(), 'Option did not have expected Name.');
$this->assertSame(1, $option->getCount(), 'Option did not have expected count.');
$option = $options['Header'];
$this->assertSame('Header', $option->getName(), 'Option did not have expected Name.');
$this->assertSame(1, $option->getCount(), 'Option did not have expected count.');
}
}

View file

@ -55,40 +55,7 @@ class FilterTest extends AbstractFunctionalTestCase
$searchRequest->setFilter(['CType' => 'HTML']); $searchRequest->setFilter(['CType' => 'HTML']);
$result = $searchService->search($searchRequest); $result = $searchService->search($searchRequest);
$this->assertSame(5, $result->getResults()[0]['uid'], 'Did not get the expected result entry.'); $this->assertSame(5, (int) $result->getResults()[0]['uid'], 'Did not get the expected result entry.');
$this->assertSame(1, count($result), 'Did not receive the single filtered element.'); $this->assertSame(1, count($result), 'Did not receive the single filtered element.');
} }
/**
* @test
*/
public function itsPossibleToFetchFacetsForField()
{
\TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class)
->get(IndexerFactory::class)
->getIndexer('tt_content')
->indexAllDocuments()
;
$searchService = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class)
->get(SearchService::class);
$searchRequest = new SearchRequest('Search Word');
$result = $searchService->search($searchRequest);
$this->assertSame(1, count($result->getFacets()), 'Did not receive the single defined facet.');
$facet = current($result->getFacets());
$this->assertSame('contentTypes', $facet->getName(), 'Name of facet was not as expected.');
$this->assertSame('CType', $facet->getField(), 'Field of facet was not expected.');
$options = $facet->getOptions();
$this->assertSame(2, count($options), 'Did not receive the expected number of possible options for facet.');
$option = $options['HTML'];
$this->assertSame('HTML', $option->getName(), 'Option did not have expected Name.');
$this->assertSame(1, $option->getCount(), 'Option did not have expected count.');
$option = $options['Header'];
$this->assertSame('Header', $option->getName(), 'Option did not have expected Name.');
$this->assertSame(1, $option->getCount(), 'Option did not have expected count.');
}
} }

View file

@ -49,12 +49,11 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
$response = $this->client->request('typo3content/_search?q=*:*'); $response = $this->client->request('typo3content/_search?q=*:*');
$this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.'); $this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.');
$this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.'); $this->assertSame($response->getData()['hits']['total'], 3, 'Not exactly 3 documents were indexed.');
$this->assertArraySubset( $this->assertSame(
['_source' => ['header' => 'indexed content element']], 'indexed content element',
$response->getData()['hits']['hits'][1], $response->getData()['hits']['hits'][2]['_source']['header'],
false,
'Record was not indexed.' 'Record was not indexed.'
); );
} }
@ -72,7 +71,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
$response = $this->client->request('typo3content/_search?q=*:*'); $response = $this->client->request('typo3content/_search?q=*:*');
$this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.'); $this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.');
$this->assertSame($response->getData()['hits']['total'], 1, 'Not exactly 1 document was indexed.'); $this->assertSame($response->getData()['hits']['total'], 1, 'Not exactly 1 document was indexed.');
$this->assertArraySubset( $this->assertArraySubset(
['_source' => ['header' => 'indexed content element']], ['_source' => ['header' => 'indexed content element']],
@ -112,8 +111,8 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
$response = $this->client->request('typo3content/_search?q=*:*'); $response = $this->client->request('typo3content/_search?q=*:*');
$this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.'); $this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.');
$this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.'); $this->assertSame($response->getData()['hits']['total'], 3, 'Not exactly 3 documents were indexed.');
} }
/** /**
@ -135,8 +134,8 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
$response = $this->client->request('typo3content/_search?q=*:*'); $response = $this->client->request('typo3content/_search?q=*:*');
$this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.'); $this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.');
$this->assertSame($response->getData()['hits']['total'], 3, 'Not exactly 3 documents were indexed.'); $this->assertSame($response->getData()['hits']['total'], 4, 'Not exactly 4 documents were indexed.');
$response = $this->client->request('typo3content/_search?q=uid:11'); $response = $this->client->request('typo3content/_search?q=uid:11');
$this->assertArraySubset( $this->assertArraySubset(
['_source' => ['header' => 'Also indexable record']], ['_source' => ['header' => 'Also indexable record']],
@ -167,8 +166,8 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
; ;
$response = $this->client->request('typo3content/_search?q=*:*'); $response = $this->client->request('typo3content/_search?q=*:*');
$this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.'); $this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.');
$this->assertSame($response->getData()['hits']['total'], 4, 'Not exactly 4 documents were indexed.'); $this->assertSame($response->getData()['hits']['total'], 5, 'Not exactly 5 documents were indexed.');
$response = $this->client->request('typo3content/_search?q=uid:11'); $response = $this->client->request('typo3content/_search?q=uid:11');
$this->assertArraySubset( $this->assertArraySubset(
@ -209,7 +208,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
/** /**
* @test * @test
*/ */
public function indexingDeltedRecordIfRecordShouldBeIndexedButIsNoLongerAvailableAndWasAlreadyIndexed() public function indexingDeletedRecordIfRecordShouldBeIndexedButIsNoLongerAvailableAndWasAlreadyIndexed()
{ {
\TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class)
->get(IndexerFactory::class) ->get(IndexerFactory::class)
@ -218,14 +217,19 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
; ;
$response = $this->client->request('typo3content/_search?q=*:*'); $response = $this->client->request('typo3content/_search?q=*:*');
$this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.'); $this->assertSame($response->getData()['hits']['total'], 3, 'Not exactly 3 documents were indexed.');
$this->getConnectionPool()->getConnectionForTable('tt_content') if ($this->isLegacyVersion()) {
->update( $this->getDatabaseConnection()
'tt_content', ->exec_UPDATEquery('tt_content', 'uid = 10', ['hidden' => 1]);
['hidden' => true], } else {
['uid' => 10] $this->getConnectionPool()->getConnectionForTable('tt_content')
); ->update(
'tt_content',
['hidden' => true],
['uid' => 10]
);
}
\TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class)
->get(IndexerFactory::class) ->get(IndexerFactory::class)
@ -234,6 +238,6 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
; ;
$response = $this->client->request('typo3content/_search?q=*:*'); $response = $this->client->request('typo3content/_search?q=*:*');
$this->assertSame($response->getData()['hits']['total'], 1, 'Not exactly 1 document is in index.'); $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 document is in index.');
} }
} }

View file

@ -0,0 +1,62 @@
<?php
namespace Codappix\SearchCore\Tests\Functional\DataProcessing;
/*
* Copyright (C) 2017 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
use Codappix\SearchCore\Compatibility\TypoScriptService76;
use Codappix\SearchCore\Compatibility\TypoScriptService;
use Codappix\SearchCore\DataProcessing\ContentObjectDataProcessorAdapterProcessor;
use Codappix\SearchCore\Tests\Functional\AbstractFunctionalTestCase;
use TYPO3\CMS\Frontend\DataProcessing\SplitProcessor;
class ContentObjectDataProcessorAdapterProcessorTest extends AbstractFunctionalTestCase
{
/**
* @test
*/
public function contentObjectDataProcessorIsExecuted()
{
$record = ['content' => 'value1, value2'];
$configuration = [
'_dataProcessor' => SplitProcessor::class,
'delimiter' => ',',
'fieldName' => 'content',
'as' => 'new_content',
];
$expectedData = [
'content' => 'value1, value2',
'new_content' => ['value1', 'value2'],
];
if ($this->isLegacyVersion()) {
$typoScriptService = new TypoScriptService76();
} else {
$typoScriptService = new TypoScriptService();
}
$subject = new ContentObjectDataProcessorAdapterProcessor($typoScriptService);
$processedData = $subject->processData($record, $configuration);
$this->assertSame(
$expectedData,
$processedData,
'The processor did not return the expected processed record.'
);
}
}

View file

@ -1,5 +1,5 @@
<?php <?php
namespace Codappix\SearchCore\Tests\Indexing\TcaIndexer; namespace Codappix\SearchCore\Tests\Functional\DataProcessing;
/* /*
* Copyright (C) 2016 Daniel Siepmann <coding@daniel-siepmann.de> * Copyright (C) 2016 Daniel Siepmann <coding@daniel-siepmann.de>
@ -20,13 +20,13 @@ namespace Codappix\SearchCore\Tests\Indexing\TcaIndexer;
* 02110-1301, USA. * 02110-1301, USA.
*/ */
use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService; use Codappix\SearchCore\DataProcessing\TcaRelationResolvingProcessor;
use Codappix\SearchCore\Tests\Functional\AbstractFunctionalTestCase; use Codappix\SearchCore\Tests\Functional\AbstractFunctionalTestCase;
use TYPO3\CMS\Backend\Utility\BackendUtility; use TYPO3\CMS\Backend\Utility\BackendUtility;
use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Extbase\Object\ObjectManager; use TYPO3\CMS\Extbase\Object\ObjectManager;
class RelationResolverTest extends AbstractFunctionalTestCase class TcaRelationResolvingProcessorTest extends AbstractFunctionalTestCase
{ {
/** /**
* @test * @test
@ -37,9 +37,8 @@ class RelationResolverTest extends AbstractFunctionalTestCase
$objectManager = GeneralUtility::makeInstance(ObjectManager::class); $objectManager = GeneralUtility::makeInstance(ObjectManager::class);
$table = 'sys_file'; $table = 'sys_file';
$subject = $objectManager->get(TcaTableService::class, $table); $subject = $objectManager->get(TcaRelationResolvingProcessor::class);
$record = BackendUtility::getRecord($table, 1); $record = $subject->processData(BackendUtility::getRecord($table, 1), ['_table' => $table]);
$subject->prepareRecord($record);
$this->assertEquals( $this->assertEquals(
[ [
@ -60,9 +59,8 @@ class RelationResolverTest extends AbstractFunctionalTestCase
$objectManager = GeneralUtility::makeInstance(ObjectManager::class); $objectManager = GeneralUtility::makeInstance(ObjectManager::class);
$table = 'tt_content'; $table = 'tt_content';
$subject = $objectManager->get(TcaTableService::class, $table); $subject = $objectManager->get(TcaRelationResolvingProcessor::class);
$record = BackendUtility::getRecord($table, 1); $record = $subject->processData(BackendUtility::getRecord($table, 1), ['_table' => $table]);
$subject->prepareRecord($record);
$this->assertEquals( $this->assertEquals(
'Insert Plugin', 'Insert Plugin',
@ -80,9 +78,8 @@ class RelationResolverTest extends AbstractFunctionalTestCase
$objectManager = GeneralUtility::makeInstance(ObjectManager::class); $objectManager = GeneralUtility::makeInstance(ObjectManager::class);
$table = 'tt_content'; $table = 'tt_content';
$subject = $objectManager->get(TcaTableService::class, $table); $subject = $objectManager->get(TcaRelationResolvingProcessor::class);
$record = BackendUtility::getRecord($table, 1); $record = $subject->processData(BackendUtility::getRecord($table, 1), ['_table' => $table]);
$subject->prepareRecord($record);
$this->assertEquals( $this->assertEquals(
[ [
@ -103,9 +100,8 @@ class RelationResolverTest extends AbstractFunctionalTestCase
$objectManager = GeneralUtility::makeInstance(ObjectManager::class); $objectManager = GeneralUtility::makeInstance(ObjectManager::class);
$table = 'tt_content'; $table = 'tt_content';
$subject = $objectManager->get(TcaTableService::class, $table); $subject = $objectManager->get(TcaRelationResolvingProcessor::class);
$record = BackendUtility::getRecord($table, 1); $record = $subject->processData(BackendUtility::getRecord($table, 1), ['_table' => $table]);
$subject->prepareRecord($record);
$this->assertEquals( $this->assertEquals(
[ [

View file

@ -15,7 +15,7 @@ plugin {
additionalWhereClause ( additionalWhereClause (
tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu', 'shortcut', 'search', 'login') tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu', 'shortcut', 'search', 'login')
AND tt_content.bodytext != '' AND (tt_content.bodytext != '' OR tt_content.header != '')
) )
mapping { mapping {
@ -23,27 +23,30 @@ plugin {
type = keyword type = keyword
} }
} }
dataProcessing {
1 = Codappix\SearchCore\DataProcessing\TcaRelationResolvingProcessor
}
} }
pages { pages {
indexer = Codappix\SearchCore\Domain\Index\TcaIndexer\PagesIndexer indexer = Codappix\SearchCore\Domain\Index\TcaIndexer\PagesIndexer
abstractFields = abstract, description, bodytext abstractFields = abstract, description, bodytext
contentFields = header, bodytext
mapping { mapping {
CType { CType {
type = keyword type = keyword
} }
} }
dataProcessing {
1 = Codappix\SearchCore\DataProcessing\TcaRelationResolvingProcessor
}
} }
} }
searching { searching {
facets {
contentTypes {
field = CType
}
}
fields { fields {
query = _all query = _all
} }

View file

@ -99,4 +99,31 @@
<colPos>0</colPos> <colPos>0</colPos>
<filelink_sorting>0</filelink_sorting> <filelink_sorting>0</filelink_sorting>
</tt_content> </tt_content>
<tt_content>
<uid>100</uid>
<pid>2</pid>
<tstamp>1480686370</tstamp>
<crdate>1480686370</crdate>
<hidden>0</hidden>
<sorting>72</sorting>
<CType>header</CType>
<header>Indexed on page 2</header>
<bodytext>This element is on a different page</bodytext>
<media>0</media>
<layout>0</layout>
<deleted>0</deleted>
<cols>0</cols>
<starttime>0</starttime>
<endtime>0</endtime>
<colPos>0</colPos>
<filelink_sorting>0</filelink_sorting>
</tt_content>
<pages>
<uid>2</uid>
<pid>1</pid>
<title>Second page with content</title>
<description>Used to check whether content is indexed only for parent page.</description>
</pages>
</dataset> </dataset>

View file

@ -0,0 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<dataset>
<tt_content>
<uid>1</uid>
<pid>1</pid>
<tstamp>1480686370</tstamp>
<crdate>1480686370</crdate>
<hidden>0</hidden>
<sorting>72</sorting>
<sys_language_uid>2</sys_language_uid>
<CType>header</CType>
<header>indexed content element</header>
<bodytext>this is the content of header content element that should get indexed</bodytext>
<media>0</media>
<layout>0</layout>
<deleted>0</deleted>
<cols>0</cols>
<starttime>0</starttime>
<endtime>0</endtime>
<colPos>0</colPos>
<filelink_sorting>0</filelink_sorting>
</tt_content>
</dataset>

View file

@ -0,0 +1,17 @@
plugin {
tx_searchcore {
settings {
searching {
facets {
contentTypes {
terms {
field = CType
}
}
}
}
}
}
}
module.tx_searchcore < plugin.tx_searchcore

View file

@ -1,8 +1,7 @@
<phpunit <phpunit
backupGlobals="true" backupGlobals="true"
backupStaticAttributes="false" backupStaticAttributes="false"
bootstrap="../../.Build/vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTestsBootstrap.php" bootstrap="Bootstrap.php"
colors="true" colors="true"
convertErrorsToExceptions="false" convertErrorsToExceptions="false"
convertWarningsToExceptions="false" convertWarningsToExceptions="false"

View file

@ -1,54 +0,0 @@
<?php
namespace Codappix\SearchCore\Tests\Functional\Hooks\DataHandler;
/*
* Copyright (C) 2016 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
use Codappix\SearchCore\Domain\Service\DataHandler as DataHandlerService;
use Codappix\SearchCore\Hook\DataHandler as DataHandlerHook;
use TYPO3\CMS\Core\DataHandling\DataHandler as Typo3DataHandler;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Extbase\Object\ObjectManager;
class IgnoresUnkownOperationTest extends AbstractDataHandlerTest
{
/**
* @var DataHandlerService|\PHPUnit_Framework_MockObject_MockObject|AccessibleObjectInterface
*/
protected $subject;
/**
* @test
*/
public function dataHandlerCommandSomethingIsIgnored()
{
$subject = new DataHandlerHook($this->subject);
$this->assertFalse(
$subject->processDatamap_afterDatabaseOperations(
'something',
'tt_content',
1,
[],
new Typo3DataHandler
),
'Hook processed status "something".'
);
}
}

Some files were not shown because too many files have changed in this diff Show more