FEATURE: Allow to configure additional where statement for indexing

* Provide TypoScript option to extend where clause to skip further
  records, e.g. tt_content by CType.
This commit is contained in:
Daniel Siepmann 2016-12-13 12:33:54 +01:00
parent fef760ee0d
commit 162d383a9d
10 changed files with 291 additions and 36 deletions

View file

@ -0,0 +1,83 @@
<?php
namespace Leonmrni\SearchCore\Configuration;
/*
* Copyright (C) 2016 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
use TYPO3\CMS\Extbase\Configuration\ConfigurationManagerInterface;
/**
* Container of all configurations for extension.
* Always inject this to have a single place for configuration and parsing only once.
*/
class ConfigurationContainer implements ConfigurationContainerInterface
{
/**
* Plaint TypoScript array from extbase for extension / plugin.
*
* @var array
*/
protected $settings;
/**
* Inject news settings via ConfigurationManager.
*
* @param ConfigurationManagerInterface $configurationManager
*/
public function injectConfigurationManager(ConfigurationManagerInterface $configurationManager)
{
$this->settings = $configurationManager->getConfiguration(
ConfigurationManagerInterface::CONFIGURATION_TYPE_SETTINGS,
'SearchCore',
'search'
);
}
/**
* @param string $section
* @param string $key
* @return mixed
* @throws InvalidArgumentException
*/
public function get($section, $key)
{
if (!isset($this->settings[$section]) || !isset($this->settings[$section][$key])) {
throw new InvalidArgumentException(
'The given configuration option does not exit.',
InvalidArgumentException::OPTION_DOES_NOT_EXIST
);
}
return $this->settings[$section][$key];
}
/**
* @param string $section
* @param string $key
* @return mixed|null
*/
public function getIfExists($section, $key)
{
try {
return $this->get($section, $key);
} catch (InvalidArgumentException $e) {
return null;
}
}
}

View file

@ -0,0 +1,49 @@
<?php
namespace Leonmrni\SearchCore\Configuration;
/*
* Copyright (C) 2016 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
use TYPO3\CMS\Core\SingletonInterface as Singleton;
/**
* Container of all configurations for extension.
* Always inject this to have a single place for configuration and parsing only once.
*/
interface ConfigurationContainerInterface extends Singleton
{
/**
* Returns the option defined by section and key.
* May throw an exception if it's not set.
*
* @param string $section
* @param string $key
* @return mixed
*/
public function get($section, $key);
/**
* Same as get but will not throw an exception but return null.
*
* @param string $section
* @param string $key
* @return mixed|null
*/
public function getIfExists($section, $key);
}

View file

@ -0,0 +1,29 @@
<?php
namespace Leonmrni\SearchCore\Configuration;
/*
* Copyright (C) 2016 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
/**
*
*/
class InvalidArgumentException extends \InvalidArgumentException
{
const OPTION_DOES_NOT_EXIST = 1481623127;
}

View file

@ -20,8 +20,8 @@ namespace Leonmrni\SearchCore\Connection\Elasticsearch;
* 02110-1301, USA.
*/
use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface;
use TYPO3\CMS\Core\SingletonInterface as Singleton;
use TYPO3\CMS\Extbase\Configuration\ConfigurationManagerInterface;
/**
* The current connection to elasticsearch.
@ -36,45 +36,24 @@ class Connection implements Singleton
protected $elasticaClient;
/**
* @var array
* @var ConfigurationContainerInterface
*/
protected $settings;
protected $configuration;
/**
* @param \Elastica\Client $elasticaClient
*/
public function __construct(\Elastica\Client $elasticaClient = null)
{
public function __construct(
\Elastica\Client $elasticaClient = null,
ConfigurationContainerInterface $configuration
) {
$this->configuration = $configuration;
$this->elasticaClient = $elasticaClient;
}
/**
* Inject news settings via ConfigurationManager.
*
* TODO: Refactor to configuration object to have a singleton holding the
* settings with validation and propper getter?
*
* @param \TYPO3\CMS\Extbase\Configuration\ConfigurationManagerInterface $configurationManager
*/
public function injectConfigurationManager(ConfigurationManagerInterface $configurationManager)
{
$this->settings = $configurationManager->getConfiguration(
ConfigurationManagerInterface::CONFIGURATION_TYPE_SETTINGS,
'SearchCore',
'search'
);
}
/**
* Used to configure elasticaClient if no one was injected. Will use
* injected settings for configuration.
*/
public function initializeObject()
{
if ($this->elasticaClient === null) {
$this->elasticaClient = new \Elastica\Client([
'host' => $this->settings['host'],
'port' => $this->settings['port'],
'host' => $this->configuration->get('connection', 'host'),
'port' => $this->configuration->get('connection', 'port'),
// TODO: Make configurable
// 'log' => 'file',
]);

View file

@ -20,8 +20,9 @@ namespace Leonmrni\SearchCore\Domain\Index\TcaIndexer;
* 02110-1301, USA.
*/
use TYPO3\CMS\Backend\Utility\BackendUtility;
use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface;
use Leonmrni\SearchCore\Domain\Index\IndexingException;
use TYPO3\CMS\Backend\Utility\BackendUtility;
/**
* Encapsulate logik related to tca configuration.
@ -40,6 +41,11 @@ class TcaTableService
*/
protected $tableName;
/**
* @var ConfigurationContaineInterfacer
*/
protected $configuration;
/**
* @var \TYPO3\CMS\Core\Log\Logger
*/
@ -55,7 +61,11 @@ class TcaTableService
$this->logger = $logManager->getLogger(__CLASS__);
}
public function __construct($tableName)
/**
* @param string $tableName
* @param ConfigurationContainer $configuration
*/
public function __construct($tableName, ConfigurationContainerInterface $configuration)
{
if (!isset($GLOBALS['TCA'][$tableName])) {
throw new IndexingException(
@ -66,6 +76,7 @@ class TcaTableService
$this->tableName = $tableName;
$this->tca = &$GLOBALS['TCA'][$this->tableName];
$this->configuration = $configuration;
}
/**
@ -114,6 +125,11 @@ class TcaTableService
. ' AND pages.no_search = 0'
;
$userDefinedWhere = $this->configuration->getIfExists('index', $this->tableName);
if (is_string($userDefinedWhere)) {
$whereClause .= $userDefinedWhere;
}
$this->logger->debug('Generated where clause.', [$this->tableName, $whereClause]);
return $whereClause;
}

View file

@ -5,6 +5,12 @@ plugin {
host = localhost
port = 9200
}
index {
tt_content {
additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu')
}
}
}
}
}

View file

@ -8,7 +8,7 @@
<hidden>0</hidden>
<sorting>72</sorting>
<CType>textmedia</CType>
<header>test</header>
<header>indexed content element</header>
<bodytext>this is the content of textmedia content element that should get indexed</bodytext>
<media>0</media>
<layout>0</layout>

View file

@ -0,0 +1,13 @@
plugin {
tx_searchcore {
settings {
index {
tt_content {
additionalWhereClause = tt_content.CType NOT IN ('div')
}
}
}
}
}
module.tx_searchcore < plugin.tx_searchcore

View file

@ -0,0 +1,42 @@
<?xml version="1.0" encoding="utf-8"?>
<dataset>
<tt_content>
<uid>9</uid>
<pid>1</pid>
<tstamp>1480686370</tstamp>
<crdate>1480686370</crdate>
<hidden>0</hidden>
<sorting>72</sorting>
<CType>textmedia</CType>
<header>Also indexable record</header>
<bodytext></bodytext>
<media>0</media>
<layout>0</layout>
<deleted>0</deleted>
<cols>0</cols>
<starttime>0</starttime>
<endtime>0</endtime>
<colPos>0</colPos>
<filelink_sorting>0</filelink_sorting>
</tt_content>
<tt_content>
<uid>10</uid>
<pid>1</pid>
<tstamp>1480686370</tstamp>
<crdate>1480686370</crdate>
<hidden>1</hidden>
<sorting>72</sorting>
<CType>div</CType>
<header>Not indexed by user where ctype condition</header>
<bodytext></bodytext>
<media>0</media>
<layout>0</layout>
<deleted>0</deleted>
<cols>0</cols>
<starttime>0</starttime>
<endtime>0</endtime>
<colPos>0</colPos>
<filelink_sorting>0</filelink_sorting>
</tt_content>
</dataset>

View file

@ -39,7 +39,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
/**
* @test
*/
public function indexBasicTtContentWithoutBasicConfiguration()
public function indexBasicTtContent()
{
\TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class)
->get(IndexerFactory::class)
@ -51,6 +51,12 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
$this->assertTrue($response->isOK());
$this->assertSame($response->getData()['hits']['total'], 1, 'Not exactly 1 document was indexed.');
$this->assertArraySubset(
['_source' => ['header' => 'indexed content element']],
$response->getData()['hits']['hits'][0],
false,
'Record was not indexed.'
);
}
/**
@ -85,4 +91,36 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
$this->assertTrue($response->isOK());
$this->assertSame($response->getData()['hits']['total'], 1, 'Not exactly 1 document was indexed.');
}
/**
* @test
*/
public function indexingRespectsUserWhereClause()
{
$this->setUpFrontendRootPage(1, ['EXT:search_core/Tests/Functional/Fixtures/Indexing/UserWhereClause.ts']);
$this->importDataSet('Tests/Functional/Fixtures/Indexing/UserWhereClause.xml');
\TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class)
->get(IndexerFactory::class)
->getIndexer('tt_content')
->index()
;
$response = $this->client->request('typo3content/_search?q=*:*');
$this->assertTrue($response->isOK());
$this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 document was indexed.');
$this->assertArraySubset(
['_source' => ['header' => 'Also indexable record']],
$response->getData()['hits']['hits'][0],
false,
'Record was not indexed.'
);
$this->assertArraySubset(
['_source' => ['header' => 'indexed content element']],
$response->getData()['hits']['hits'][1],
false,
'Record was not indexed.'
);
}
}