From 2b4a3a5bd6b0548274b74ce8ced4ce9c763214e9 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 13 Dec 2016 16:43:51 +0100 Subject: [PATCH 1/6] FEATURE: Enable hooks --- ext_localconf.php | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/ext_localconf.php b/ext_localconf.php index 91f7230..458d21e 100644 --- a/ext_localconf.php +++ b/ext_localconf.php @@ -14,15 +14,14 @@ call_user_func( Leonmrni\SearchCore\Command\IndexCommandController::class, ], ], - // Not yet, first finish whole indexing through command controller as it's more important. - // 't3lib/class.t3lib_tcemain.php' => [ - // 'processCmdmapClass' => [ - // $extensionKey => \Leonmrni\SearchCore\Hook\DataHandler::class, - // ], - // 'processDatamapClass' => [ - // $extensionKey => \Leonmrni\SearchCore\Hook\DataHandler::class, - // ], - // ], + 't3lib/class.t3lib_tcemain.php' => [ + 'processCmdmapClass' => [ + $extensionKey => \Leonmrni\SearchCore\Hook\DataHandler::class, + ], + 'processDatamapClass' => [ + $extensionKey => \Leonmrni\SearchCore\Hook\DataHandler::class, + ], + ], ], ] ); From 0953f4bb1f312307b05a47084284c6e7c4cbe732 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 13 Dec 2016 16:55:57 +0100 Subject: [PATCH 2/6] FEATURE: Index resolved relations * TCAIndexer is now able to resolve relations of any kind by using TYPO3 Core API. * Indexed will be a single string or an array, depending of how many relations were resolved. * The same value will be indexed as shown by TCA in backend while editing or displaying. --- .../InvalidArgumentException.php | 3 - .../TcaIndexer/InvalidArgumentException.php | 27 ++++ .../Index/TcaIndexer/RelationResolver.php | 119 +++++++++++++++++ .../Index/TcaIndexer/TcaTableService.php | 44 ++++--- .../Functional/AbstractFunctionalTestCase.php | 2 + .../Fixtures/Indexing/ResolveRelations.xml | 123 ++++++++++++++++++ .../Functional/Indexing/IndexTcaTableTest.php | 47 +++++++ 7 files changed, 347 insertions(+), 18 deletions(-) create mode 100644 Classes/Domain/Index/TcaIndexer/InvalidArgumentException.php create mode 100644 Classes/Domain/Index/TcaIndexer/RelationResolver.php create mode 100644 Tests/Functional/Fixtures/Indexing/ResolveRelations.xml diff --git a/Classes/Configuration/InvalidArgumentException.php b/Classes/Configuration/InvalidArgumentException.php index 078485a..fd9e048 100644 --- a/Classes/Configuration/InvalidArgumentException.php +++ b/Classes/Configuration/InvalidArgumentException.php @@ -20,9 +20,6 @@ namespace Leonmrni\SearchCore\Configuration; * 02110-1301, USA. */ -/** - * - */ class InvalidArgumentException extends \InvalidArgumentException { const OPTION_DOES_NOT_EXIST = 1481623127; diff --git a/Classes/Domain/Index/TcaIndexer/InvalidArgumentException.php b/Classes/Domain/Index/TcaIndexer/InvalidArgumentException.php new file mode 100644 index 0000000..0f3dc9a --- /dev/null +++ b/Classes/Domain/Index/TcaIndexer/InvalidArgumentException.php @@ -0,0 +1,27 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +class InvalidArgumentException extends \InvalidArgumentException +{ + const COLUMN_DOES_NOT_EXIST = 1481632388; + const RECORD_NOT_FOUND = 1481643208; +} diff --git a/Classes/Domain/Index/TcaIndexer/RelationResolver.php b/Classes/Domain/Index/TcaIndexer/RelationResolver.php new file mode 100644 index 0000000..fe8dc4c --- /dev/null +++ b/Classes/Domain/Index/TcaIndexer/RelationResolver.php @@ -0,0 +1,119 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use TYPO3\CMS\Backend\Utility\BackendUtility; +use TYPO3\CMS\Core\SingletonInterface as Singleton; + +/** + * Resolves relations from TCA using TCA. + * + * E.g. resolves mm relations, items for selects, group db, etc. + * Will replace the column with an array of resolved labels. + */ +class RelationResolver implements Singleton +{ + /** + * @var \TYPO3\CMS\Backend\Form\DataPreprocessor + * @inject + */ + protected $formEngine; + + /** + * Resolve relations for the given record. + * + * @param TcaTableService $service + * @param array $record + */ + public function resolveRelationsForRecord(TcaTableService $service, array &$record) + { + $preprocessedData = $this->formEngine->renderRecordRaw( + $service->getTableName(), + $record['uid'], + $record['pid'], + $record + ); + + foreach (array_keys($record) as $column) { + try { + $config = $service->getColumnConfig($column); + } catch (InvalidArgumentException $e) { + // Column is not configured. + continue; + } + + if (! $this->isRelation($config)) { + continue; + } + + $record[$column] = $this->resolveValue( + $preprocessedData[$column], + $config, + $column + ); + } + } + + /** + * Resolve the given value from TYPO3 API response. + * + * As FormEngine uses an internal format, we resolve it to a usable format + * for indexing. + * + * TODO: Unittest to break as soon as TYPO3 api has changed, so we know + * exactly that we need to tackle this place. + * + * @param string $value The value from FormEngine to resolve. + * + * @return array|string + */ + protected function resolveValue($value) + { + $newValue = []; + if ($value === '' || $value === '0') { + return ''; + } + + if (strpos($value, '|') === false) { + return $value; + } + + foreach (\TYPO3\CMS\Core\Utility\GeneralUtility::trimExplode(',', $value) as $value) { + $value = substr($value, strpos($value, '|') + 1); + $value = rawurldecode($value); + $newValue[] = $value; + } + + return $newValue; + } + + /** + * @param array Column config. + * @return bool + */ + protected function isRelation(array &$config) + { + return isset($config['foreign_table']) + || (isset($config['items']) && is_array($config['items'])) + || (isset($config['internal_type']) && strtolower($config['internal_type']) === 'db') + ; + } +} diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index b717169..3350e6b 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -25,7 +25,7 @@ use Leonmrni\SearchCore\Domain\Index\IndexingException; use TYPO3\CMS\Backend\Utility\BackendUtility; /** - * Encapsulate logik related to tca configuration. + * Encapsulate logik related to TCA configuration. */ class TcaTableService { @@ -51,6 +51,11 @@ class TcaTableService */ protected $logger; + /** + * @var RelationResolver + */ + protected $relationResolver; + /** * Inject log manager to get concrete logger from it. * @@ -65,8 +70,11 @@ class TcaTableService * @param string $tableName * @param ConfigurationContainer $configuration */ - public function __construct($tableName, ConfigurationContainerInterface $configuration) - { + public function __construct( + $tableName, + RelationResolver $relationResolver, + ConfigurationContainerInterface $configuration + ) { if (!isset($GLOBALS['TCA'][$tableName])) { throw new IndexingException( 'Table "' . $tableName . '" is not configured in TCA.', @@ -77,6 +85,7 @@ class TcaTableService $this->tableName = $tableName; $this->tca = &$GLOBALS['TCA'][$this->tableName]; $this->configuration = $configuration; + $this->relationResolver = $relationResolver; } /** @@ -101,7 +110,7 @@ class TcaTableService */ public function prepareRecord(array &$record) { - // TODO: Resolve values from 'items' like static select, radio or checkbox. + $this->relationResolver->resolveRelationsForRecord($this, $record); if (isset($record['uid']) && !isset($record['search_identifier'])) { $record['search_identifier'] = $record['uid']; @@ -144,8 +153,7 @@ class TcaTableService array_filter( array_keys($this->tca['columns']), function ($columnName) { - $columnConfig = $this->tca['columns'][$columnName]['config']; - return !$this->isRelation($columnConfig) && !$this->isSystemField($columnName); + return !$this->isSystemField($columnName); } ) ); @@ -158,15 +166,6 @@ class TcaTableService return implode(',', $fields); } - /** - * @param array - * @return bool - */ - protected function isRelation(array &$columnConfig) - { - return isset($columnConfig['foreign_table']); - } - /** * @param string * @return bool @@ -189,4 +188,19 @@ class TcaTableService return in_array($columnName, $systemFields); } + + /** + * @param string $columnName + * @return array + * @throws InvalidArgumentException + */ + public function getColumnConfig($columnName) + { + if (!isset($this->tca['columns'][$columnName])) { + throw new InvalidArgumentException('Column does not exist.', InvalidArgumentException::COLUMN_DOES_NOT_EXIST); + } + + return $this->tca['columns'][$columnName]['config']; + } + } diff --git a/Tests/Functional/AbstractFunctionalTestCase.php b/Tests/Functional/AbstractFunctionalTestCase.php index fd90e8e..65b698f 100644 --- a/Tests/Functional/AbstractFunctionalTestCase.php +++ b/Tests/Functional/AbstractFunctionalTestCase.php @@ -40,6 +40,8 @@ abstract class AbstractFunctionalTestCase extends CoreTestCase { parent::setUp(); + \TYPO3\CMS\Core\Core\Bootstrap::getInstance()->initializeLanguageObject(); + // Provide necessary configuration for extension $this->importDataSet('Tests/Functional/Fixtures/BasicSetup.xml'); $this->setUpFrontendRootPage(1, ['EXT:search_core/Tests/Functional/Fixtures/BasicSetup.ts']); diff --git a/Tests/Functional/Fixtures/Indexing/ResolveRelations.xml b/Tests/Functional/Fixtures/Indexing/ResolveRelations.xml new file mode 100644 index 0000000..1c4b5cd --- /dev/null +++ b/Tests/Functional/Fixtures/Indexing/ResolveRelations.xml @@ -0,0 +1,123 @@ + + + + 9 + 1 + 1480686370 + 1480686370 + 0 + 72 + textmedia +
Record with relation to multiple sys category record
+ some content + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 3 + 0 + + + + 10 + 1 + 1480686370 + 1480686370 + 0 + 72 + textmedia +
Record with relation to a single sys category record
+ some content + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + 0 + + + + 1 + 1 + 1480686370 + 1480686370 + 0 + 0 + 0 + 1 + Category 1 + Category for testing + 0 + 1 + + + + 2 + 1 + 1480686370 + 1480686370 + 0 + 0 + 0 + 1 + Category 2 + Another category for testing + 1 + 1 + + + + 3 + 1 + 1480686370 + 1480686370 + 1 + 0 + 0 + 1 + Deleted category + + 0 + 1 + + + + 1 + 9 + tt_content + categories + 2 + 2 + + + 2 + 9 + tt_content + categories + 1 + 1 + + + 3 + 9 + tt_content + categories + 3 + 3 + + + + 2 + 10 + tt_content + categories + 1 + 1 + + diff --git a/Tests/Functional/Indexing/IndexTcaTableTest.php b/Tests/Functional/Indexing/IndexTcaTableTest.php index 88ba1e4..4d5f981 100644 --- a/Tests/Functional/Indexing/IndexTcaTableTest.php +++ b/Tests/Functional/Indexing/IndexTcaTableTest.php @@ -123,4 +123,51 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase 'Record was not indexed.' ); } + + /** + * @test + */ + public function resolvesRelations() + { + $this->setUpBackendUserFromFixture(1); + $this->importDataSet('Tests/Functional/Fixtures/Indexing/ResolveRelations.xml'); + + \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) + ->get(IndexerFactory::class) + ->getIndexer('tt_content') + ->index() + ; + + $response = $this->client->request('typo3content/_search?q=*:*'); + + $this->assertArraySubset( + ['_source' => [ + 'uid' => '9', + 'CType' => 'textmedia', // Testing items + 'categories' => ['Category 2', 'Category 1'], // Testing mm (with sorting) + ]], + $response->getData()['hits']['hits'][0], + false, + 'Record was not indexed with resolved category relation to a single value.' + ); + $this->assertArraySubset( + ['_source' => [ + 'uid' => '10', + 'CType' => 'textmedia', + 'categories' => ['Category 2'], + ]], + $response->getData()['hits']['hits'][1], + false, + 'Record was not indexed with resolved category relation to multiple values.' + ); + $this->assertArraySubset( + ['_source' => [ + 'uid' => '6', + 'categories' => null, + ]], + $response->getData()['hits']['hits'][2], + false, + 'Record was indexed with resolved category relation, but should not have any.' + ); + } } From 1e7566139553e97c9eaa2e31e70c8c85d62e8bd1 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 13 Dec 2016 17:10:23 +0100 Subject: [PATCH 3/6] BUGFIX: Fix broken tests * Always provide backend user for tests, as we always add language handling. * Only this way all relations can be resolved in all tests. --- Tests/Functional/AbstractFunctionalTestCase.php | 1 + Tests/Functional/Indexing/IndexTcaTableTest.php | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/Functional/AbstractFunctionalTestCase.php b/Tests/Functional/AbstractFunctionalTestCase.php index 65b698f..d0d0257 100644 --- a/Tests/Functional/AbstractFunctionalTestCase.php +++ b/Tests/Functional/AbstractFunctionalTestCase.php @@ -40,6 +40,7 @@ abstract class AbstractFunctionalTestCase extends CoreTestCase { parent::setUp(); + $this->setUpBackendUserFromFixture(1); \TYPO3\CMS\Core\Core\Bootstrap::getInstance()->initializeLanguageObject(); // Provide necessary configuration for extension diff --git a/Tests/Functional/Indexing/IndexTcaTableTest.php b/Tests/Functional/Indexing/IndexTcaTableTest.php index 4d5f981..6d0999b 100644 --- a/Tests/Functional/Indexing/IndexTcaTableTest.php +++ b/Tests/Functional/Indexing/IndexTcaTableTest.php @@ -129,7 +129,6 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase */ public function resolvesRelations() { - $this->setUpBackendUserFromFixture(1); $this->importDataSet('Tests/Functional/Fixtures/Indexing/ResolveRelations.xml'); \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) From 99fbb28b9d9a3f1c045b9312453b4a2f56efe8a3 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 13 Dec 2016 17:19:02 +0100 Subject: [PATCH 4/6] CLEANUP: Keep line short --- Classes/Domain/Index/TcaIndexer/TcaTableService.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index 3350e6b..cc808dc 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -197,7 +197,10 @@ class TcaTableService public function getColumnConfig($columnName) { if (!isset($this->tca['columns'][$columnName])) { - throw new InvalidArgumentException('Column does not exist.', InvalidArgumentException::COLUMN_DOES_NOT_EXIST); + throw new InvalidArgumentException( + 'Column does not exist.', + InvalidArgumentException::COLUMN_DOES_NOT_EXIST + ); } return $this->tca['columns'][$columnName]['config']; From a87610716b35966cac46a03fb41bc82165d7f643 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 13 Dec 2016 17:51:30 +0100 Subject: [PATCH 5/6] FEATURE: Improve tests * As they fail on travis and have a dependency to result order. * Resolve the dependency and query each document itself. --- Tests/Functional/Indexing/IndexTcaTableTest.php | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/Tests/Functional/Indexing/IndexTcaTableTest.php b/Tests/Functional/Indexing/IndexTcaTableTest.php index 6d0999b..64fb55a 100644 --- a/Tests/Functional/Indexing/IndexTcaTableTest.php +++ b/Tests/Functional/Indexing/IndexTcaTableTest.php @@ -109,7 +109,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase $response = $this->client->request('typo3content/_search?q=*:*'); $this->assertTrue($response->isOK()); - $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 document was indexed.'); + $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.'); $this->assertArraySubset( ['_source' => ['header' => 'Also indexable record']], $response->getData()['hits']['hits'][0], @@ -138,7 +138,10 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase ; $response = $this->client->request('typo3content/_search?q=*:*'); + $this->assertTrue($response->isOK()); + $this->assertSame($response->getData()['hits']['total'], 3, 'Not exactly 3 documents were indexed.'); + $response = $this->client->request('typo3content/_search?q=uid:9'); $this->assertArraySubset( ['_source' => [ 'uid' => '9', @@ -147,24 +150,28 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase ]], $response->getData()['hits']['hits'][0], false, - 'Record was not indexed with resolved category relation to a single value.' + 'Record was not indexed with resolved category relations to multiple values.' ); + + $response = $this->client->request('typo3content/_search?q=uid:10'); $this->assertArraySubset( ['_source' => [ 'uid' => '10', 'CType' => 'textmedia', 'categories' => ['Category 2'], ]], - $response->getData()['hits']['hits'][1], + $response->getData()['hits']['hits'][0], false, - 'Record was not indexed with resolved category relation to multiple values.' + 'Record was not indexed with resolved category relations to a single value.' ); + + $response = $this->client->request('typo3content/_search?q=uid:6'); $this->assertArraySubset( ['_source' => [ 'uid' => '6', 'categories' => null, ]], - $response->getData()['hits']['hits'][2], + $response->getData()['hits']['hits'][0], false, 'Record was indexed with resolved category relation, but should not have any.' ); From dc7a8f77bffd7c31b6fcbd20106a83d9d478d929 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 15 Dec 2016 09:01:05 +0100 Subject: [PATCH 6/6] TASK: Remove TYPO3 7 * As relation resolver is not compatible yet. See branch feature/resolve-relations/typo3-7 --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 539e746..1b76868 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,8 +17,6 @@ env: - typo3DatabasePassword="" matrix: - TYPO3_VERSION="~6.2" - - TYPO3_VERSION="~7.6" - - TYPO3_VERSION="dev-master" matrix: fast_finish: true