FEATURE: Allow custom indexer

Adjust code to use defined indexer as FQCN, to allow any class to be
used as indexer. Also classes defined by user.
This commit is contained in:
Daniel Siepmann 2017-07-04 12:12:36 +02:00
parent 03a953c13f
commit 432335c80d
Signed by: Daniel Siepmann
GPG key ID: 33D6629915560EF4
9 changed files with 255 additions and 154 deletions

View file

@ -21,6 +21,7 @@ namespace Leonmrni\SearchCore\Command;
*/ */
use Leonmrni\SearchCore\Domain\Index\IndexerFactory; use Leonmrni\SearchCore\Domain\Index\IndexerFactory;
use Leonmrni\SearchCore\Domain\Index\NoMatchingIndexerException;
use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Extbase\Mvc\Controller\CommandController; use TYPO3\CMS\Extbase\Mvc\Controller\CommandController;
@ -34,12 +35,6 @@ class IndexCommandController extends CommandController
*/ */
protected $indexerFactory; protected $indexerFactory;
/**
* @var \Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface
* @inject
*/
protected $configuration;
/** /**
* @param IndexerFactory $factory * @param IndexerFactory $factory
*/ */
@ -49,19 +44,18 @@ class IndexCommandController extends CommandController
} }
/** /**
* Will index the given table or everything. * Will index the given identifier.
* *
* @param string $table * @param string $identifier
*/ */
public function indexCommand($table) public function indexCommand($identifier)
{ {
// TODO: Allow to index multiple tables at once?
// TODO: Also allow to index everything? // TODO: Also allow to index everything?
if ($this->configuration->getIfExists('indexing.' . $table) === null) { try {
$this->outputLine('Table is not allowed for indexing.'); $this->indexerFactory->getIndexer($identifier)->indexAllDocuments();
$this->quit(1); $this->outputLine($identifier . ' was indexed.');
} catch (NoMatchingIndexerException $e) {
$this->outputLine('No indexer found for: ' . $identifier);
} }
$this->indexerFactory->getIndexer($table)->indexAllDocuments();
$this->outputLine('Table was indexed.');
} }
} }

View file

@ -0,0 +1,113 @@
<?php
namespace Leonmrni\SearchCore\Domain\Index;
/*
* Copyright (C) 2017 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
use Leonmrni\SearchCore\Connection\ConnectionInterface;
abstract class AbstractIndexer implements IndexerInterface
{
/**
* @var ConnectionInterface
*/
protected $connection;
/**
* @var \TYPO3\CMS\Core\Log\Logger
*/
protected $logger;
/**
* Inject log manager to get concrete logger from it.
*
* @param \TYPO3\CMS\Core\Log\LogManager $logManager
*/
public function injectLogger(\TYPO3\CMS\Core\Log\LogManager $logManager)
{
$this->logger = $logManager->getLogger(__CLASS__);
}
/**
* @param ConnectionInterface $connection
*/
public function __construct(ConnectionInterface $connection)
{
$this->connection = $connection;
}
public function indexAllDocuments()
{
$this->logger->info('Start indexing');
foreach ($this->getRecordGenerator() as $records) {
$this->logger->debug('Index records.', [$records]);
if ($records === null) {
break;
}
$this->connection->addDocuments($this->getDocumentName(), $records);
}
$this->logger->info('Finish indexing');
}
public function indexDocument($identifier)
{
$this->logger->info('Start indexing single record.', [$identifier]);
try {
$this->connection->addDocument($this->getDocumentName(), $this->getRecord($identifier));
} catch (NoRecordFoundException $e) {
$this->logger->info('Could not index document.', [$e->getMessage()]);
}
$this->logger->info('Finish indexing');
}
/**
* @return \Generator
*/
protected function getRecordGenerator()
{
$offset = 0;
// TODO: Make configurable.
$limit = 50;
while (($records = $this->getRecords($offset, $limit)) !== []) {
yield $records;
$offset += $limit;
}
}
/**
* @param int $offset
* @param int $limit
* @return array|null
*/
abstract protected function getRecords($offset, $limit);
/**
* @param int $identifier
* @return array
* @throws NoRecordFoundException If record could not be found.
*/
abstract protected function getRecord($identifier);
/**
* @return string
*/
abstract protected function getDocumentName();
}

View file

@ -20,6 +20,11 @@ namespace Leonmrni\SearchCore\Domain\Index;
* 02110-1301, USA. * 02110-1301, USA.
*/ */
use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface;
use Leonmrni\SearchCore\Configuration\InvalidArgumentException;
use Leonmrni\SearchCore\Domain\Index\IndexerInterface;
use Leonmrni\SearchCore\Domain\Index\TcaIndexer;
use Leonmrni\SearchCore\Domain\Index\TcaIndexer\TcaTableService;
use TYPO3\CMS\Core\SingletonInterface as Singleton; use TYPO3\CMS\Core\SingletonInterface as Singleton;
use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; use TYPO3\CMS\Extbase\Object\ObjectManagerInterface;
@ -33,28 +38,61 @@ class IndexerFactory implements Singleton
*/ */
protected $objectManager; protected $objectManager;
/**
* @var ConfigurationContainerInterface
*/
protected $configuration;
/** /**
* @param ObjectManagerInterface $objectManager * @param ObjectManagerInterface $objectManager
*/ */
public function __construct(ObjectManagerInterface $objectManager) public function __construct(
{ ObjectManagerInterface $objectManager,
ConfigurationContainerInterface $configuration
) {
$this->objectManager = $objectManager; $this->objectManager = $objectManager;
$this->configuration = $configuration;
} }
/** /**
* @param string $tableName * @param string $identifier
* *
* @return IndexerInterface * @return IndexerInterface
* @throws NoMatchingIndexer
*/ */
public function getIndexer($tableName) public function getIndexer($identifier)
{ {
// This is the place to use configuration to return different indexer. try {
return $this->buildIndexer($this->configuration->get('indexing.' . $identifier . '.indexer'), $identifier);
} catch (NoMatchingIndexerException $e) {
// Nothing to do, we throw exception below
} catch (InvalidArgumentException $e) {
// Nothing to do, we throw exception below
}
throw new NoMatchingIndexerException('Could not find an indexer for ' . $identifier, 1497341442);
}
/**
* @param string $indexerClass
* @param string $identifier
*
* @return IndexerInterface
* @throws NoMatchingIndexer
*/
protected function buildIndexer($indexerClass, $identifier)
{
if ($indexerClass === TcaIndexer::class) {
return $this->objectManager->get( return $this->objectManager->get(
TcaIndexer::Class, TcaIndexer::class,
$this->objectManager->get( $this->objectManager->get(TcaTableService::class, $identifier)
TcaIndexer\TcaTableService::class,
$tableName
)
); );
} }
if (class_exists($indexerClass) && in_array(IndexerInterface::class, class_implements($indexerClass))) {
return $this->objectManager->get($indexerClass);
}
throw new NoMatchingIndexerException('Could not find indexer: ' . $indexerClass, 1497341442);
}
} }

View file

@ -0,0 +1,25 @@
<?php
namespace Leonmrni\SearchCore\Domain\Index;
/*
* Copyright (C) 2017 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
class NoMatchingIndexerException extends IndexingException
{
}

View file

@ -26,33 +26,13 @@ use Leonmrni\SearchCore\Connection\ConnectionInterface;
/** /**
* Will index the given table using configuration from TCA. * Will index the given table using configuration from TCA.
*/ */
class TcaIndexer implements IndexerInterface class TcaIndexer extends AbstractIndexer
{ {
/**
* @var ConnectionInterface
*/
protected $connection;
/** /**
* @var TcaIndexer\TcaTableService * @var TcaIndexer\TcaTableService
*/ */
protected $tcaTableService; protected $tcaTableService;
/**
* @var \TYPO3\CMS\Core\Log\Logger
*/
protected $logger;
/**
* Inject log manager to get concrete logger from it.
*
* @param \TYPO3\CMS\Core\Log\LogManager $logManager
*/
public function injectLogger(\TYPO3\CMS\Core\Log\LogManager $logManager)
{
$this->logger = $logManager->getLogger(__CLASS__);
}
/** /**
* @param TcaIndexer\TcaTableService $tcaTableService * @param TcaIndexer\TcaTableService $tcaTableService
* @param ConnectionInterface $connection * @param ConnectionInterface $connection
@ -65,46 +45,6 @@ class TcaIndexer implements IndexerInterface
$this->connection = $connection; $this->connection = $connection;
} }
public function indexAllDocuments()
{
$this->logger->info('Start indexing');
foreach ($this->getRecordGenerator() as $records) {
$this->logger->debug('Index records.', [$records]);
if ($records === null) {
break;
}
$this->connection->addDocuments($this->tcaTableService->getTableName(), $records);
}
$this->logger->info('Finish indexing');
}
public function indexDocument($identifier)
{
$this->logger->info('Start indexing single record.', [$identifier]);
try {
$this->connection->addDocument($this->tcaTableService->getTableName(), $this->getRecord($identifier));
} catch (NoRecordFoundException $e) {
$this->logger->info('Could not index document.', [$e->getMessage()]);
}
$this->logger->info('Finish indexing');
}
/**
* @return \Generator
*/
protected function getRecordGenerator()
{
$offset = 0;
// TODO: Make configurable.
$limit = 50;
while (($records = $this->getRecords($offset, $limit)) !== []) {
yield $records;
$offset += $limit;
}
}
/** /**
* @param int $offset * @param int $offset
* @param int $limit * @param int $limit
@ -156,4 +96,12 @@ class TcaIndexer implements IndexerInterface
return $record; return $record;
} }
/**
* @return string
*/
protected function getDocumentName()
{
return $this->tcaTableService->getTableName();
}
} }

View file

@ -21,6 +21,8 @@ namespace Leonmrni\SearchCore\Domain\Service;
*/ */
use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface;
use Leonmrni\SearchCore\Domain\Index\IndexerFactory;
use Leonmrni\SearchCore\Domain\Index\NoMatchingIndexerException;
use Leonmrni\SearchCore\Domain\Index\TcaIndexer; use Leonmrni\SearchCore\Domain\Index\TcaIndexer;
use TYPO3\CMS\Core\SingletonInterface as Singleton; use TYPO3\CMS\Core\SingletonInterface as Singleton;
use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Core\Utility\GeneralUtility;
@ -47,8 +49,7 @@ class DataHandler implements Singleton
protected $connection; protected $connection;
/** /**
* @var \Leonmrni\SearchCore\Domain\Index\IndexerFactory * @var IndexerFactory
* @inject
*/ */
protected $indexerFactory; protected $indexerFactory;
@ -74,30 +75,12 @@ class DataHandler implements Singleton
/** /**
* @param ConfigurationContainerInterface $configuration * @param ConfigurationContainerInterface $configuration
* @param IndexerFactory $indexerFactory
*/ */
public function __construct(ConfigurationContainerInterface $configuration) public function __construct(ConfigurationContainerInterface $configuration, IndexerFactory $indexerFactory)
{ {
$this->configuration = $configuration; $this->configuration = $configuration;
} $this->indexerFactory = $indexerFactory;
/**
* Get all tables that are allowed for indexing.
*
* @return array<String>
*/
public function getAllowedTablesForIndexing()
{
$tables = [];
foreach ($this->configuration->get('indexing') as $tableName => $indexConfiguration) {
if ($indexConfiguration['indexer'] === TcaIndexer::class) {
$tables[] = $tableName;
continue;
}
// TODO: Support custom indexer.
// Define "interface" / option which is used.
}
return $tables;
} }
/** /**
@ -107,7 +90,7 @@ class DataHandler implements Singleton
public function add($table, array $record) public function add($table, array $record)
{ {
$this->logger->debug('Record received for add.', [$table, $record]); $this->logger->debug('Record received for add.', [$table, $record]);
$this->indexerFactory->getIndexer($table)->indexDocument($record['uid']); $this->getIndexer($table)->indexDocument($record['uid']);
} }
/** /**
@ -116,7 +99,7 @@ class DataHandler implements Singleton
public function update($table, array $record) public function update($table, array $record)
{ {
$this->logger->debug('Record received for update.', [$table, $record]); $this->logger->debug('Record received for update.', [$table, $record]);
$this->indexerFactory->getIndexer($table)->indexDocument($record['uid']); $this->getIndexer($table)->indexDocument($record['uid']);
} }
/** /**
@ -128,4 +111,31 @@ class DataHandler implements Singleton
$this->logger->debug('Record received for delete.', [$table, $identifier]); $this->logger->debug('Record received for delete.', [$table, $identifier]);
$this->connection->deleteDocument($table, $identifier); $this->connection->deleteDocument($table, $identifier);
} }
/**
* @param string $table
* @return IndexerInterface
*
* @throws NoMatchingIndexerException
*/
protected function getIndexer($table)
{
return $this->indexerFactory->getIndexer($table);
}
/**
* @param string $table
* @return bool
*/
public function canHandle($table)
{
try {
$this->getIndexer($table);
return true;
} catch (NoMatchingIndexerException $e) {
return false;
}
return false;
}
} }

View file

@ -21,6 +21,7 @@ namespace Leonmrni\SearchCore\Hook;
*/ */
use Leonmrni\SearchCore\Configuration\NoConfigurationException; use Leonmrni\SearchCore\Configuration\NoConfigurationException;
use Leonmrni\SearchCore\Domain\Index\NoMatchingIndexerException;
use Leonmrni\SearchCore\Domain\Service\DataHandler as OwnDataHandler; use Leonmrni\SearchCore\Domain\Service\DataHandler as OwnDataHandler;
use TYPO3\CMS\Backend\Utility\BackendUtility; use TYPO3\CMS\Backend\Utility\BackendUtility;
use TYPO3\CMS\Core\DataHandling\DataHandler as CoreDataHandler; use TYPO3\CMS\Core\DataHandling\DataHandler as CoreDataHandler;
@ -140,7 +141,7 @@ class DataHandler implements Singleton
$this->logger->debug('Datahandler could not be setup.'); $this->logger->debug('Datahandler could not be setup.');
return false; return false;
} }
if (! $this->shouldProcessTable($table)) { if (! $this->dataHandler->canHandle($table)) {
$this->logger->debug('Table is not allowed.', [$table]); $this->logger->debug('Table is not allowed.', [$table]);
return false; return false;
} }
@ -148,15 +149,6 @@ class DataHandler implements Singleton
return true; return true;
} }
/**
* @param string $table
* @return bool
*/
protected function shouldProcessTable($table)
{
return in_array($table, $this->dataHandler->getAllowedTablesForIndexing());
}
/** /**
* Wrapper to allow unit testing. * Wrapper to allow unit testing.
* *

View file

@ -21,6 +21,7 @@ namespace Leonmrni\SearchCore\Tests\Functional\Hooks\DataHandler;
*/ */
use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface;
use Leonmrni\SearchCore\Domain\Index\IndexerFactory;
use Leonmrni\SearchCore\Domain\Service\DataHandler as DataHandlerService; use Leonmrni\SearchCore\Domain\Service\DataHandler as DataHandlerService;
use Leonmrni\SearchCore\Hook\DataHandler as DataHandlerHook; use Leonmrni\SearchCore\Hook\DataHandler as DataHandlerHook;
use Leonmrni\SearchCore\Tests\Functional\AbstractFunctionalTestCase; use Leonmrni\SearchCore\Tests\Functional\AbstractFunctionalTestCase;
@ -42,7 +43,10 @@ abstract class AbstractDataHandlerTest extends AbstractFunctionalTestCase
$objectManager = GeneralUtility::makeInstance(ObjectManager::class); $objectManager = GeneralUtility::makeInstance(ObjectManager::class);
$this->subject = $this->getMockBuilder(DataHandlerService::class) $this->subject = $this->getMockBuilder(DataHandlerService::class)
->setConstructorArgs([$objectManager->get(ConfigurationContainerInterface::class)]) ->setConstructorArgs([
$objectManager->get(ConfigurationContainerInterface::class),
$objectManager->get(IndexerFactory::class)
])
->setMethods(['add', 'update', 'delete']) ->setMethods(['add', 'update', 'delete'])
->getMock(); ->getMock();

View file

@ -21,8 +21,8 @@ namespace Leonmrni\SearchCore\Tests\Unit\Command;
*/ */
use Leonmrni\SearchCore\Command\IndexCommandController; use Leonmrni\SearchCore\Command\IndexCommandController;
use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface;
use Leonmrni\SearchCore\Domain\Index\IndexerFactory; use Leonmrni\SearchCore\Domain\Index\IndexerFactory;
use Leonmrni\SearchCore\Domain\Index\NoMatchingIndexerException;
use Leonmrni\SearchCore\Domain\Index\TcaIndexer; use Leonmrni\SearchCore\Domain\Index\TcaIndexer;
use Leonmrni\SearchCore\Tests\Unit\AbstractUnitTestCase; use Leonmrni\SearchCore\Tests\Unit\AbstractUnitTestCase;
use TYPO3\CMS\Extbase\Mvc\Controller\CommandController; use TYPO3\CMS\Extbase\Mvc\Controller\CommandController;
@ -40,11 +40,6 @@ class IndexCommandControllerTest extends AbstractUnitTestCase
*/ */
protected $indexerFactory; protected $indexerFactory;
/**
* @var ConfigurationContainerInterface
*/
protected $configuration;
public function setUp() public function setUp()
{ {
parent::setUp(); parent::setUp();
@ -52,16 +47,12 @@ class IndexCommandControllerTest extends AbstractUnitTestCase
$this->indexerFactory = $this->getMockBuilder(IndexerFactory::class) $this->indexerFactory = $this->getMockBuilder(IndexerFactory::class)
->disableOriginalConstructor() ->disableOriginalConstructor()
->getMock(); ->getMock();
$this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)
->disableOriginalConstructor()
->getMock();
$this->subject = $this->getMockBuilder(IndexCommandController::class) $this->subject = $this->getMockBuilder(IndexCommandController::class)
->disableOriginalConstructor() ->disableOriginalConstructor()
->setMethods(['quit', 'outputLine']) ->setMethods(['quit', 'outputLine'])
->getMock(); ->getMock();
$this->subject->injectIndexerFactory($this->indexerFactory); $this->subject->injectIndexerFactory($this->indexerFactory);
$this->inject($this->subject, 'configuration', $this->configuration);
} }
/** /**
@ -69,22 +60,14 @@ class IndexCommandControllerTest extends AbstractUnitTestCase
*/ */
public function indexerStopsForNonAllowedTable() public function indexerStopsForNonAllowedTable()
{ {
$this->expectException(StopActionException::class);
$this->subject->expects($this->once())
->method('quit')
->with(1)
->will($this->throwException(new StopActionException));
$this->subject->expects($this->once()) $this->subject->expects($this->once())
->method('outputLine') ->method('outputLine')
->with('Table is not allowed for indexing.'); ->with('No indexer found for: nonAllowedTable');
$this->indexerFactory->expects($this->never()) $this->indexerFactory->expects($this->once())
->method('getIndexer'); ->method('getIndexer')
->with('nonAllowedTable')
->will($this->throwException(new NoMatchingIndexerException));
$this->configuration->expects($this->once())
->method('getIfExists')
->with('indexing.nonAllowedTable')
->will($this->returnValue(null));
$this->subject->indexCommand('nonAllowedTable'); $this->subject->indexCommand('nonAllowedTable');
} }
@ -100,18 +83,12 @@ class IndexCommandControllerTest extends AbstractUnitTestCase
->method('quit'); ->method('quit');
$this->subject->expects($this->once()) $this->subject->expects($this->once())
->method('outputLine') ->method('outputLine')
->with('Table was indexed.'); ->with('allowedTable was indexed.');
$this->indexerFactory->expects($this->once()) $this->indexerFactory->expects($this->once())
->method('getIndexer') ->method('getIndexer')
->with('allowedTable') ->with('allowedTable')
->will($this->returnValue($indexerMock)); ->will($this->returnValue($indexerMock));
$this->configuration->expects($this->once())
->method('getIfExists')
->with('indexing.allowedTable')
->will($this->returnValue([
'indexer' => 'Leonmrni\SearchCore\Domain\Index\TcaIndexer',
]));
$this->subject->indexCommand('allowedTable'); $this->subject->indexCommand('allowedTable');
} }
} }