Work-Related/search_core
Work-Related/search_core
1
0
mirror of https://github.com/Codappix/search_core.git synced 2024-11-23 18:56:12 +01:00
Code Releases Activity

TASK: Add changes from customer project

Browse source
This commit is contained in:
Daniel Siepmann 2017-10-10 11:15:06 +02:00
parent 250dd25107
commit aebe58721d
Signed by: Daniel Siepmann
GPG key ID: 33D6629915560EF4
15 changed files with 270 additions and 103 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
Classes/Connection/Elasticsearch/IndexFactory.php
View file

@ -39,6 +39,21 @@ class IndexFactory implements Singleton
*/
protected $configuration;
/**
* @var \TYPO3\CMS\Core\Log\Logger
*/
protected $logger;
/**
* Inject log manager to get concrete logger from it.
*
* @param \TYPO3\CMS\Core\Log\LogManager $logManager
*/
public function injectLogger(\TYPO3\CMS\Core\Log\LogManager $logManager)
{
$this->logger = $logManager->getLogger(__CLASS__);
}
/**
* @param ConfigurationContainerInterface $configuration
*/
@ -60,7 +75,10 @@ class IndexFactory implements Singleton
$index = $connection->getClient()->getIndex('typo3content');
if ($index->exists() === false) {
$index->create($this->getConfigurationFor($documentType));
$config = $this->getConfigurationFor($documentType);
$this->logger->debug(sprintf('Create index %s.', $documentType), [$documentType, $config]);
$index->create($config);
$this->logger->debug(sprintf('Created index %s.', $documentType), [$documentType]);
}
return $index;
@ -76,9 +94,11 @@ class IndexFactory implements Singleton
try {
$configuration = $this->configuration->get('indexing.' . $documentType . '.index');
if (isset($configuration['analysis']['analyzer'])) {
foreach ($configuration['analysis']['analyzer'] as $key => $analyzer) {
$configuration['analysis']['analyzer'][$key] = $this->prepareAnalyzerConfiguration($analyzer);
foreach (['analyzer', 'filter'] as $optionsToExpand) {
if (isset($configuration['analysis'][$optionsToExpand])) {
foreach ($configuration['analysis'][$optionsToExpand] as $key => $options) {
$configuration['analysis'][$optionsToExpand][$key] = $this->prepareOptions($options);
}
}
}
@ -89,20 +109,20 @@ class IndexFactory implements Singleton
}
/**
* @param array $analyzer
* @param array $options
*
* @return array
*/
protected function prepareAnalyzerConfiguration(array $analyzer)
protected function prepareOptions(array $options)
{
$fieldsToExplode = ['char_filter', 'filter'];
$fieldsToExplode = ['char_filter', 'filter', 'word_list'];
foreach ($fieldsToExplode as $fieldToExplode) {
if (isset($analyzer[$fieldToExplode])) {
$analyzer[$fieldToExplode] = GeneralUtility::trimExplode(',', $analyzer[$fieldToExplode], true);
if (isset($options[$fieldToExplode])) {
$options[$fieldToExplode] = GeneralUtility::trimExplode(',', $options[$fieldToExplode], true);
}
}
return $analyzer;
return $options;
}
}

2
Classes/Connection/Elasticsearch/SearchResult.php
View file

@ -83,7 +83,7 @@ class SearchResult implements SearchResultInterface
/**
* Return all facets, if any.
*
* @return array<FacetIterface>
* @return array<FacetInterface>
*/
public function getFacets()
{

2
Classes/Connection/SearchResultInterface.php
View file

@ -35,7 +35,7 @@ interface SearchResultInterface extends \Iterator, \Countable, QueryResultInterf
/**
* Return all facets, if any.
*
* @return array<FacetIterface>
* @return array<FacetInterface>
*/
public function getFacets();

54
Classes/DataProcessing/CopyToProcessor.php Normal file
View file

@ -0,0 +1,54 @@
<?php
namespace Codappix\SearchCore\DataProcessing;
/*
* Copyright (C) 2017 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
/**
* Copies values from one field to another one.
*/
class CopyToProcessor implements ProcessorInterface
{
public function processRecord(array $record, array $configuration)
{
$all = [];
$this->addArray($all, $record);
$all = array_filter($all);
$record[$configuration['to']] = implode(PHP_EOL, $all);
return $record;
}
/**
* @param array &$to
* @param array $from
*/
protected function addArray(array &$to, array $from)
{
foreach ($from as $property => $value) {
if (is_array($value)) {
$this->addArray($to, $value, $exclude);
continue;
}
$to[] = (string) $value;
}
}
}

39
Classes/DataProcessing/ProcessorInterface.php Normal file
View file

@ -0,0 +1,39 @@
<?php
namespace Codappix\SearchCore\DataProcessing;
/*
* Copyright (C) 2017 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
/**
* All DataProcessing Processor should implement this interface, otherwise they
* will not be executed.
*/
interface ProcessorInterface
{
/**
* Processes the given record.
* Also retrieves the configuration for this processor instance.
*
* @param array $record
* @param array $configuration
*
* @return array
*/
public function processRecord(array $record, array $configuration);
}

40
Classes/Domain/Index/TcaIndexer/PagesIndexer.php
View file

@ -34,6 +34,12 @@ class PagesIndexer extends TcaIndexer
*/
protected $contentTableService;
/**
* @var \TYPO3\CMS\Core\Resource\FileRepository
* @inject
*/
protected $fileRepository;
/**
* @param TcaTableService $tcaTableService
* @param TcaTableService $tcaTableService
@ -65,7 +71,9 @@ class PagesIndexer extends TcaIndexer
}
}
$record['content'] = $this->fetchContentForPage($record['uid']);
$content = $this->fetchContentForPage($record['uid']);
$record['content'] = $content['content'];
$record['media'] = array_unique(array_merge($record['media'], $content['images']));
parent::prepareRecord($record);
}
@ -88,14 +96,42 @@ class PagesIndexer extends TcaIndexer
}
$this->logger->debug('Fetched content for page ' . $uid);
$images = [];
$content = [];
foreach ($contentElements as $contentElement) {
$images = array_merge(
$images,
$this->getContentElementImages($contentElement['uid'])
);
$content[] = $contentElement['bodytext'];
}
return [
// Remove Tags.
// Interpret escaped new lines and special chars.
// Trim, e.g. trailing or leading new lines.
return trim(stripcslashes(strip_tags(implode(' ', $content))));
'content' => trim(stripcslashes(strip_tags(implode(' ', $content)))),
'images' => $images,
];
}
/**
* @param int $uidOfContentElement
* @return array
*/
protected function getContentElementImages($uidOfContentElement)
{
$imageRelationUids = [];
$imageRelations = $this->fileRepository->findByRelation(
'tt_content',
'image',
$uidOfContentElement
);
foreach ($imageRelations as $relation) {
$imageRelationUids[] = $relation->getUid();
}
return $imageRelationUids;
}
}

14
Classes/Domain/Index/TcaIndexer/RelationResolver.php
View file

@ -86,6 +86,9 @@ class RelationResolver implements Singleton
*/
protected function resolveValue($value, array $config)
{
if (isset($config['foreign_table']) && $config['foreign_table'] === 'sys_file_reference') {
return $this->resolveFalRelations($value);
}
if ($value === '' || $value === '0') {
return '';
}
@ -102,6 +105,17 @@ class RelationResolver implements Singleton
return '';
}
/**
* @param string $value
* @return array
*/
protected function resolveFalRelations($value)
{
$files = GeneralUtility::trimExplode(',', $value);
$files = array_filter($files);
return array_map('intval', $files);
}
/**
* @param array Column config.
* @return bool

13
Classes/Domain/Index/TcaIndexer/TcaTableService.php
View file

@ -21,6 +21,8 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer;
*/
use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
use Codappix\SearchCore\Configuration\InvalidArgumentException as InvalidConfigurationArgumentException;
use Codappix\SearchCore\DataProcessing\ProcessorInterface;
use Codappix\SearchCore\Domain\Index\IndexingException;
use TYPO3\CMS\Backend\Utility\BackendUtility;
use TYPO3\CMS\Core\Utility\GeneralUtility;
@ -148,6 +150,17 @@ class TcaTableService
{
$this->relationResolver->resolveRelationsForRecord($this, $record);
try {
foreach ($this->configuration->get('indexing.' . $this->tableName . '.dataProcessing') as $configuration) {
$dataProcessor = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance($configuration['_typoScriptNodeValue']);
if ($dataProcessor instanceof ProcessorInterface) {
$record = $dataProcessor->processRecord($record, $configuration);
}
}
} catch (InvalidConfigurationArgumentException $e) {
// Nothing to do.
}
if (isset($record['uid']) && !isset($record['search_identifier'])) {
$record['search_identifier'] = $record['uid'];
}

21
Classes/Domain/Search/QueryFactory.php
View file

@ -25,6 +25,7 @@ use Codappix\SearchCore\Configuration\InvalidArgumentException;
use Codappix\SearchCore\Connection\ConnectionInterface;
use Codappix\SearchCore\Connection\Elasticsearch\Query;
use Codappix\SearchCore\Connection\SearchRequestInterface;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Extbase\Utility\ArrayUtility;
class QueryFactory
@ -109,20 +110,18 @@ class QueryFactory
return;
}
$query = ArrayUtility::setValueByPath(
$query,
'query.bool.must.0.match._all.query',
$searchRequest->getSearchTerm()
);
$matchExpression = [
'type' => 'most_fields',
'query' => $searchRequest->getSearchTerm(),
'fields' => GeneralUtility::trimExplode(',', $this->configuration->get('searching.fields')),
];
$minimumShouldMatch = $this->configuration->getIfExists('searching.minimumShouldMatch');
if ($minimumShouldMatch) {
$query = ArrayUtility::setValueByPath(
$query,
'query.bool.must.0.match._all.minimum_should_match',
$minimumShouldMatch
);
$matchExpression['minimum_should_match'] = $minimumShouldMatch;
}
$query = ArrayUtility::setValueByPath($query, 'query.bool.must.0.multi_match', $matchExpression);
}
/**
@ -150,6 +149,7 @@ class QueryFactory
];
}
if (!empty($boostQueryParts)) {
$query = ArrayUtility::arrayMergeRecursiveOverrule($query, [
'query' => [
'bool' => [
@ -158,6 +158,7 @@ class QueryFactory
],
]);
}
}
/**
* @param array $query

4
Tests/Functional/Connection/Elasticsearch/FilterTest.php
View file

@ -48,7 +48,7 @@ class FilterTest extends AbstractFunctionalTestCase
$searchService = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class)
->get(SearchService::class);
$searchRequest = new SearchRequest('Search Word');
$searchRequest = new SearchRequest();
$result = $searchService->search($searchRequest);
$this->assertSame(2, count($result), 'Did not receive both indexed elements without filter.');
@ -73,7 +73,7 @@ class FilterTest extends AbstractFunctionalTestCase
$searchService = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class)
->get(SearchService::class);
$searchRequest = new SearchRequest('Search Word');
$searchRequest = new SearchRequest();
$result = $searchService->search($searchRequest);
$this->assertSame(1, count($result->getFacets()), 'Did not receive the single defined facet.');

1
Tests/Functional/Fixtures/BasicSetup.ts
View file

@ -37,6 +37,7 @@ plugin {
}
searching {
fields = search_title
facets {
contentTypes {
field = CType

1
Tests/Unit/Connection/Elasticsearch/IndexFactoryTest.php
View file

@ -38,6 +38,7 @@ class IndexFactoryTest extends AbstractUnitTestCase
$this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock();
$this->subject = new IndexFactory($this->configuration);
$this->subject->injectLogger($this->getMockedLogger());
}
/**

20
Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php
View file

@ -60,18 +60,10 @@ class TcaTableServiceTest extends AbstractUnitTestCase
->method('getIfExists')
->withConsecutive(['indexing.table.additionalWhereClause'], ['indexing.table.rootLineBlacklist'])
->will($this->onConsecutiveCalls(null, false));
$this->subject->expects($this->once())
->method('getSystemWhereClause')
->will($this->returnValue('1=1 AND pages.no_search = 0'));
$whereClause = $this->subject->getWhereClause();
$this->assertSame(
'1=1 AND pages.no_search = 0',
$whereClause->getStatement()
);
$this->assertSame(
[],
$whereClause->getParameters()
$this->subject->getWhereClause()
);
}
@ -84,18 +76,10 @@ class TcaTableServiceTest extends AbstractUnitTestCase
->method('getIfExists')
->withConsecutive(['indexing.table.additionalWhereClause'], ['indexing.table.rootLineBlacklist'])
->will($this->onConsecutiveCalls('table.field = "someValue"', false));
$this->subject->expects($this->once())
->method('getSystemWhereClause')
->will($this->returnValue('1=1 AND pages.no_search = 0'));
$whereClause = $this->subject->getWhereClause();
$this->assertSame(
'1=1 AND pages.no_search = 0 AND table.field = "someValue"',
$whereClause->getStatement()
);
$this->assertSame(
[],
$whereClause->getParameters()
$this->subject->getWhereClause()
);
}
}

96
Tests/Unit/Domain/Search/QueryFactoryTest.php
View file

@ -50,13 +50,11 @@ class QueryFactoryTest extends AbstractUnitTestCase
/**
* @test
*/
public function creatonOfQueryWorksInGeneral()
public function creationOfQueryWorksInGeneral()
{
$searchRequest = new SearchRequest('SearchWord');
$this->mockConfiguration();
$this->configuration->expects($this->any())
->method('get')
->will($this->throwException(new InvalidArgumentException));
$searchRequest = new SearchRequest('SearchWord');
$query = $this->subject->create($searchRequest);
$this->assertInstanceOf(
@ -71,9 +69,7 @@ class QueryFactoryTest extends AbstractUnitTestCase
*/
public function filterIsAddedToQuery()
{
$this->configuration->expects($this->any())
->method('get')
->will($this->throwException(new InvalidArgumentException));
$this->mockConfiguration();
$searchRequest = new SearchRequest('SearchWord');
$searchRequest->setFilter(['field' => 'content']);
@ -83,7 +79,7 @@ class QueryFactoryTest extends AbstractUnitTestCase
[
['term' => ['field' => 'content']]
],
$query->toArray()['query']['bool']['filter'],
$query->toArray()['query']['function_score']['query']['bool']['filter'],
'Filter was not added to query.'
);
}
@ -93,9 +89,7 @@ class QueryFactoryTest extends AbstractUnitTestCase
*/
public function emptyFilterIsNotAddedToQuery()
{
$this->configuration->expects($this->any())
->method('get')
->will($this->throwException(new InvalidArgumentException));
$this->mockConfiguration();
$searchRequest = new SearchRequest('SearchWord');
$searchRequest->setFilter([
@ -112,7 +106,7 @@ class QueryFactoryTest extends AbstractUnitTestCase
$query = $this->subject->create($searchRequest);
$this->assertSame(
null,
$query->toArray()['query']['bool']['filter'],
$query->toArray()['query']['function_score']['query']['bool']['filter'],
'Filter was added to query, even if no filter exists.'
);
}
@ -122,9 +116,8 @@ class QueryFactoryTest extends AbstractUnitTestCase
*/
public function facetsAreAddedToQuery()
{
$this->configuration->expects($this->any())
->method('get')
->will($this->throwException(new InvalidArgumentException));
$this->mockConfiguration();
$searchRequest = new SearchRequest('SearchWord');
$searchRequest->addFacet(new FacetRequest('Identifier', 'FieldName'));
$searchRequest->addFacet(new FacetRequest('Identifier 2', 'FieldName 2'));
@ -153,9 +146,8 @@ class QueryFactoryTest extends AbstractUnitTestCase
*/
public function sizeIsAddedToQuery()
{
$this->configuration->expects($this->any())
->method('get')
->will($this->throwException(new InvalidArgumentException));
$this->mockConfiguration();
$searchRequest = new SearchRequest('SearchWord');
$searchRequest->setLimit(45);
$searchRequest->setOffset(35);
@ -178,10 +170,9 @@ class QueryFactoryTest extends AbstractUnitTestCase
*/
public function searchTermIsAddedToQuery()
{
$this->mockConfiguration();
$searchRequest = new SearchRequest('SearchWord');
$this->configuration->expects($this->any())
->method('get')
->will($this->throwException(new InvalidArgumentException));
$query = $this->subject->create($searchRequest);
$this->assertSame(
@ -189,16 +180,18 @@ class QueryFactoryTest extends AbstractUnitTestCase
'bool' => [
'must' => [
[
'match' => [
'_all' => [
'multi_match' => [
'type' => 'most_fields',
'query' => 'SearchWord',
'fields' => [
0 => 'test_field',
],
],
],
],
],
],
$query->toArray()['query'],
$query->toArray()['query']['function_score']['query'],
'Search term was not added to query as expected.'
);
}
@ -208,14 +201,13 @@ class QueryFactoryTest extends AbstractUnitTestCase
*/
public function minimumShouldMatchIsAddedToQuery()
{
$searchRequest = new SearchRequest('SearchWord');
$this->configuration->expects($this->once())
->method('getIfExists')
->with('searching.minimumShouldMatch')
->willReturn('50%');
$this->configuration->expects($this->any())
->method('get')
->will($this->throwException(new InvalidArgumentException));
$this->mockConfiguration();
$searchRequest = new SearchRequest('SearchWord');
$query = $this->subject->create($searchRequest);
$this->assertArraySubset(
@ -223,16 +215,14 @@ class QueryFactoryTest extends AbstractUnitTestCase
'bool' => [
'must' => [
[
'match' => [
'_all' => [
'multi_match' => [
'minimum_should_match' => '50%',
],
],
],
],
],
],
$query->toArray()['query'],
$query->toArray()['query']['function_score']['query'],
'minimum_should_match was not added to query as configured.'
);
}
@ -244,10 +234,11 @@ class QueryFactoryTest extends AbstractUnitTestCase
{
$searchRequest = new SearchRequest('SearchWord');
$this->configuration->expects($this->exactly(2))
$this->configuration->expects($this->exactly(3))
->method('get')
->withConsecutive(['searching.boost'], ['searching.fieldValueFactor'])
->withConsecutive(['searching.fields'], ['searching.boost'], ['searching.fieldValueFactor'])
->will($this->onConsecutiveCalls(
'test_field',
[
'search_title' => 3,
'search_abstract' => 1.5,
@ -292,10 +283,11 @@ class QueryFactoryTest extends AbstractUnitTestCase
'factor' => '2',
'missing' => '1',
];
$this->configuration->expects($this->exactly(2))
$this->configuration->expects($this->exactly(3))
->method('get')
->withConsecutive(['searching.boost'], ['searching.fieldValueFactor'])
->withConsecutive(['searching.fields'], ['searching.boost'], ['searching.fieldValueFactor'])
->will($this->onConsecutiveCalls(
'test_field',
$this->throwException(new InvalidArgumentException),
$fieldConfig
));
@ -308,9 +300,11 @@ class QueryFactoryTest extends AbstractUnitTestCase
'bool' => [
'must' => [
[
'match' => [
'_all' => [
'multi_match' => [
'type' => 'most_fields',
'query' => 'SearchWord',
'fields' => [
0 => 'test_field',
],
],
],
@ -330,17 +324,27 @@ class QueryFactoryTest extends AbstractUnitTestCase
*/
public function emptySearchStringWillNotAddSearchToQuery()
{
$this->mockConfiguration();
$searchRequest = new SearchRequest();
$this->configuration->expects($this->any())
->method('get')
->will($this->throwException(new InvalidArgumentException));
$query = $this->subject->create($searchRequest);
$this->assertInstanceOf(
stdClass,
$query->toArray()['query']['match_all'],
$this->assertNull(
$query->toArray()['query']['function_score']['query'],
'Empty search request does not create expected query.'
);
}
protected function mockConfiguration()
{
$this->configuration->expects($this->any())
->method('get')
->will($this->returnCallback(function ($option) {
if ($option === 'searching.fields') {
return 'test_field';
}
return $this->throwException(new InvalidArgumentException);
}));
}
}

2
ext_localconf.php
View file

@ -11,7 +11,7 @@ call_user_func(
'SC_OPTIONS' => [
'extbase' => [
'commandControllers' => [
Codappix\SearchCore\Command\IndexCommandController::class,
$extensionKey . '::index' => Codappix\SearchCore\Command\IndexCommandController::class,
],
],
't3lib/class.t3lib_tcemain.php' => [