From e297d556e68ad8abf35524920cf822068b40c1be Mon Sep 17 00:00:00 2001 From: Benjamin Serfhos Date: Thu, 11 Oct 2018 16:50:32 +0200 Subject: [PATCH] [!!!][FEATURE] Remap specific document type's to ES6 identifiers Multiple types are not allowed any more in ES6. Refactored to use document type and custom identifier to still create different 'types'. See: https://www.elastic.co/guide/en/elasticsearch/reference/6.x/removal-of-types.html --- Classes/Command/IndexCommandController.php | 30 ++++++++---- Classes/Connection/ConnectionInterface.php | 13 ++++- Classes/Connection/Elasticsearch.php | 47 +++++++++++++------ .../Elasticsearch/DocumentFactory.php | 1 - Classes/Domain/Index/AbstractIndexer.php | 22 ++++++++- .../Index/TcaIndexer/TcaTableService.php | 3 -- 6 files changed, 84 insertions(+), 32 deletions(-) diff --git a/Classes/Command/IndexCommandController.php b/Classes/Command/IndexCommandController.php index de66c11..d148dee 100644 --- a/Classes/Command/IndexCommandController.php +++ b/Classes/Command/IndexCommandController.php @@ -23,6 +23,7 @@ namespace Codappix\SearchCore\Command; use Codappix\SearchCore\Domain\Index\IndexerFactory; use Codappix\SearchCore\Domain\Index\NoMatchingIndexerException; +use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Extbase\Mvc\Controller\CommandController; /** @@ -52,12 +53,17 @@ class IndexCommandController extends CommandController */ public function indexCommand(string $identifier) { - try { - $this->indexerFactory->getIndexer($identifier)->indexAllDocuments(); - $this->outputLine($identifier . ' was indexed.'); - } catch (NoMatchingIndexerException $e) { - $this->outputLine('No indexer found for: ' . $identifier); + // Allow multiple identifiers per import task + $identifiers = GeneralUtility::trimExplode(',', $identifier, true); + foreach ($identifiers as $value) { + try { + $this->indexerFactory->getIndexer($value)->indexAllDocuments(); + $this->outputLine($value . ' was indexed.'); + } catch (NoMatchingIndexerException $e) { + $this->outputLine('No indexer found for: ' . $value); + } } + } /** @@ -68,11 +74,15 @@ class IndexCommandController extends CommandController */ public function deleteCommand(string $identifier) { - try { - $this->indexerFactory->getIndexer($identifier)->delete(); - $this->outputLine($identifier . ' was deleted.'); - } catch (NoMatchingIndexerException $e) { - $this->outputLine('No indexer found for: ' . $identifier); + // Allow multiple identifiers per import task + $identifiers = GeneralUtility::trimExplode(',', $identifier, true); + foreach ($identifiers as $value) { + try { + $this->indexerFactory->getIndexer($value)->delete(); + $this->outputLine($value . ' was deleted.'); + } catch (NoMatchingIndexerException $e) { + $this->outputLine('No indexer found for: ' . $value); + } } } } diff --git a/Classes/Connection/ConnectionInterface.php b/Classes/Connection/ConnectionInterface.php index b8af94a..bad861c 100644 --- a/Classes/Connection/ConnectionInterface.php +++ b/Classes/Connection/ConnectionInterface.php @@ -21,6 +21,8 @@ namespace Codappix\SearchCore\Connection; * 02110-1301, USA. */ +use Elastica\Query; + /** * Defines interface for connections to storage backend for interacting with documents. */ @@ -77,8 +79,15 @@ interface ConnectionInterface /** * Will delete the whole index / db. * - * @param string $documentType * @return void */ - public function deleteIndex(string $documentType); + public function deleteIndex(); + + /** + * Will delete the index / db of defined document type. + * + * @param Query $query + * @return void + */ + public function deleteIndexByQuery(Query $query); } diff --git a/Classes/Connection/Elasticsearch.php b/Classes/Connection/Elasticsearch.php index bf85b5d..98c12f0 100644 --- a/Classes/Connection/Elasticsearch.php +++ b/Classes/Connection/Elasticsearch.php @@ -23,8 +23,10 @@ namespace Codappix\SearchCore\Connection; use Codappix\SearchCore\Connection\Elasticsearch\SearchResult; use Codappix\SearchCore\Domain\Search\QueryFactory; +use Elastica\Query; use TYPO3\CMS\Core\SingletonInterface as Singleton; use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; +use TYPO3\CMS\Extbase\Utility\DebuggerUtility; /** * Outer wrapper to elasticsearch. @@ -121,8 +123,8 @@ class Elasticsearch implements Singleton, ConnectionInterface { $this->withType( $documentType, - function ($type) use ($document) { - $type->addDocument($this->documentFactory->getDocument($type->getName(), $document)); + function ($type) use ($documentType, $document) { + $type->addDocument($this->documentFactory->getDocument($documentType, $document)); } ); } @@ -156,8 +158,8 @@ class Elasticsearch implements Singleton, ConnectionInterface { $this->withType( $documentType, - function ($type) use ($document) { - $type->updateDocument($this->documentFactory->getDocument($type->getName(), $document)); + function ($type) use ($documentType, $document) { + $type->updateDocument($this->documentFactory->getDocument($documentType, $document)); } ); } @@ -170,23 +172,23 @@ class Elasticsearch implements Singleton, ConnectionInterface { $this->withType( $documentType, - function ($type) use ($documents) { - $type->addDocuments($this->documentFactory->getDocuments($type->getName(), $documents)); + function ($type) use ($documentType, $documents) { + $type->addDocuments($this->documentFactory->getDocuments($documentType, $documents)); } ); } /** - * @param string $documentType + * @return void */ - public function deleteIndex(string $documentType) + public function deleteIndex() { $index = $this->connection->getClient()->getIndex($this->indexFactory->getIndexName()); if (!$index->exists()) { $this->logger->notice( 'Index did not exist, therefore was not deleted.', - [$documentType, $this->indexFactory->getIndexName()] + [$this->indexFactory->getIndexName()] ); return; } @@ -194,6 +196,24 @@ class Elasticsearch implements Singleton, ConnectionInterface $index->delete(); } + /** + * @param Query $query + * @return void + */ + public function deleteIndexByQuery(Query $query) + { + $index = $this->connection->getClient()->getIndex($this->indexFactory->getIndexName()); + if (!$index->exists()) { + $this->logger->notice( + 'Index did not exist, therefore items can not be deleted by query.', + [$this->indexFactory->getIndexName(), $query->getQuery()] + ); + return; + } + + $index->deleteByQuery($query); + } + /** * Execute given callback with Elastica Type based on provided documentType * @@ -202,7 +222,7 @@ class Elasticsearch implements Singleton, ConnectionInterface */ protected function withType(string $documentType, callable $callback) { - $type = $this->getType($documentType); + $type = $this->getType(); // TODO: Check whether it's to heavy to send it so often e.g. for every single document. // Perhaps add command controller to submit mapping?! // Also it's not possible to change mapping without deleting index first. @@ -230,17 +250,16 @@ class Elasticsearch implements Singleton, ConnectionInterface } /** - * @param string $documentType * @return \Elastica\Type */ - protected function getType(string $documentType): \Elastica\Type + protected function getType(): \Elastica\Type { return $this->typeFactory->getType( $this->indexFactory->getIndex( $this->connection, - $documentType + 'document' ), - $documentType + 'document' ); } } diff --git a/Classes/Connection/Elasticsearch/DocumentFactory.php b/Classes/Connection/Elasticsearch/DocumentFactory.php index 797bf0b..086fff3 100644 --- a/Classes/Connection/Elasticsearch/DocumentFactory.php +++ b/Classes/Connection/Elasticsearch/DocumentFactory.php @@ -60,7 +60,6 @@ class DocumentFactory implements Singleton } $identifier = $document['search_identifier']; - unset($document['search_identifier']); $this->logger->debug( sprintf('Convert %s %u to document.', $documentType, $identifier), diff --git a/Classes/Domain/Index/AbstractIndexer.php b/Classes/Domain/Index/AbstractIndexer.php index 12c2e5d..37e2f29 100644 --- a/Classes/Domain/Index/AbstractIndexer.php +++ b/Classes/Domain/Index/AbstractIndexer.php @@ -24,6 +24,7 @@ namespace Codappix\SearchCore\Domain\Index; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Connection\ConnectionInterface; +use Elastica\Query; use TYPO3\CMS\Core\Utility\GeneralUtility; abstract class AbstractIndexer implements IndexerInterface @@ -130,7 +131,13 @@ abstract class AbstractIndexer implements IndexerInterface public function delete() { $this->logger->info('Start deletion of index.'); - $this->connection->deleteIndex($this->getDocumentName()); + $this->connection->deleteIndexByQuery(Query::create([ + 'query' => [ + 'regexp' => [ + 'search_identifier' => $this->getDocumentName() . '-*' + ] + ] + ])); $this->logger->info('Finish deletion.'); } @@ -161,10 +168,21 @@ abstract class AbstractIndexer implements IndexerInterface } catch (InvalidArgumentException $e) { // Nothing to do. } - + $this->generateSearchIdentifier($record); $this->handleAbstract($record); } + /** + * @param array $record + * @return void + */ + protected function generateSearchIdentifier(array &$record) + { + if (!isset($record['search_identifier']) && isset($record['uid'])) { + $record['search_identifier'] = $this->getDocumentName() . '-' . $record['uid']; + } + } + /** * @param array $record * @return void diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index 297d0fb..1d6ea90 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -168,9 +168,6 @@ class TcaTableService implements TcaTableServiceInterface */ public function prepareRecord(array &$record) { - if (isset($record['uid']) && !isset($record['search_identifier'])) { - $record['search_identifier'] = $record['uid']; - } if (isset($record[$this->tca['ctrl']['label']]) && !isset($record['search_title'])) { $record['search_title'] = $record[$this->tca['ctrl']['label']]; }