Merge pull request #146 from Codappix/release/0.0.3

Release: 0.0.3
This commit is contained in:
Daniel Siepmann 2018-03-22 14:21:49 +01:00 committed by GitHub
commit 7cc1616d3a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
27 changed files with 314 additions and 63 deletions

View file

@ -108,7 +108,7 @@ class SearchResult implements SearchResultInterface
}
foreach ($this->result->getResults() as $result) {
$this->results[] = new ResultItem($result->getData());
$this->results[] = new ResultItem($result->getData(), $result->getParam('_type'));
}
}

View file

@ -33,4 +33,12 @@ interface ResultItemInterface extends \ArrayAccess
* Used e.g. for dataprocessing.
*/
public function getPlainData() : array;
/**
* Returns the type of the item.
*
* That should make it easier to differentiate if multiple
* types are returned for one query.
*/
public function getType() : string;
}

View file

@ -27,11 +27,21 @@ class CopyToProcessor implements ProcessorInterface
{
public function processData(array $record, array $configuration) : array
{
$all = [];
$target = [];
$this->addArray($all, $record);
$all = array_filter($all);
$record[$configuration['to']] = implode(PHP_EOL, $all);
$from = $record;
if (isset($configuration['from'])) {
$from = $record[$configuration['from']];
}
if (is_array($from)) {
$this->addArray($target, $from);
} else {
$target[] = (string) $from;
}
$target = array_filter($target);
$record[$configuration['to']] = implode(PHP_EOL, $target);
return $record;
}

View file

@ -24,6 +24,7 @@ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
use Codappix\SearchCore\Connection\ConnectionInterface;
use Codappix\SearchCore\Domain\Index\TcaIndexer;
use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService;
use TYPO3\CMS\Core\Utility\GeneralUtility;
/**
* Specific indexer for Pages, will basically add content of page.
@ -79,8 +80,14 @@ class PagesIndexer extends TcaIndexer
protected function fetchContentForPage(int $uid) : array
{
if ($this->contentTableService instanceof TcaTableService) {
$contentElements = $this->contentTableService->getQuery()
->execute()->fetchAll();
$queryBuilder = $this->contentTableService->getQuery();
$queryBuilder->andWhere(
$queryBuilder->expr()->eq(
$this->contentTableService->getTableName() . '.pid',
$queryBuilder->createNamedParameter($uid, \PDO::PARAM_INT)
)
);
$contentElements = $queryBuilder->execute()->fetchAll();
} else {
$contentElements = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows(
$this->contentTableService->getFields(),
@ -103,7 +110,7 @@ class PagesIndexer extends TcaIndexer
$images,
$this->getContentElementImages($contentElement['uid'])
);
$content[] = $contentElement['bodytext'];
$content[] = $this->getContentFromContentElement($contentElement);
}
return [
@ -136,4 +143,22 @@ class PagesIndexer extends TcaIndexer
return $imageRelationUids;
}
protected function getContentFromContentElement(array $contentElement) : string
{
$content = '';
$fieldsWithContent = GeneralUtility::trimExplode(
',',
$this->configuration->get('indexing.' . $this->identifier . '.contentFields'),
true
);
foreach ($fieldsWithContent as $fieldWithContent) {
if (isset($contentElement[$fieldWithContent]) && trim($contentElement[$fieldWithContent])) {
$content .= trim($contentElement[$fieldWithContent]) . ' ';
}
}
return trim($content);
}
}

View file

@ -29,9 +29,20 @@ class ResultItem implements ResultItemInterface
*/
protected $data = [];
public function __construct(array $result)
/**
* @var string
*/
protected $type = '';
public function __construct(array $result, string $type)
{
$this->data = $result;
$this->type = $type;
}
public function getType() : string
{
return $this->type;
}
public function getPlainData() : array

View file

@ -76,7 +76,7 @@ class SearchResult implements SearchResultInterface
}
foreach ($this->resultItems as $item) {
$this->results[] = new ResultItem($item);
$this->results[] = new ResultItem($item['data'], $item['type']);
}
}

View file

@ -146,10 +146,13 @@ class SearchService
$newSearchResultItems = [];
foreach ($this->configuration->get('searching.dataProcessing') as $configuration) {
foreach ($searchResult as $resultItem) {
$newSearchResultItems[] = $this->dataProcessorService->executeDataProcessor(
$configuration,
$resultItem->getPlainData()
);
$newSearchResultItems[] = [
'data' => $this->dataProcessorService->executeDataProcessor(
$configuration,
$resultItem->getPlainData()
),
'type' => $resultItem->getType(),
];
}
}

View file

@ -72,7 +72,7 @@ class DataHandler implements Singleton
/**
* Called by CoreDataHandler on deletion of records.
*/
public function processCmdmap_deleteAction(string $table, int $uid) : bool
public function processCmdmap_deleteAction(string $table, string $uid) : bool
{
if (! $this->shouldProcessHookForTable($table)) {
$this->logger->debug('Delete not processed.', [$table, $uid]);
@ -95,6 +95,10 @@ class DataHandler implements Singleton
$uid = $dataHandler->substNEWwithIDs[$uid];
}
if (!is_numeric($uid) || $uid <= 0) {
continue;
}
$this->processRecord($table, $uid);
}
}

View file

@ -10,12 +10,13 @@ plugin {
indexing {
tt_content {
additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu', 'shortcut', 'search', 'login') AND tt_content.bodytext != ''
additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu', 'shortcut', 'search', 'login') AND (tt_content.bodytext != '' OR tt_content.header != '')
}
pages {
additionalWhereClause = pages.doktype NOT IN (3, 199, 6, 254, 255)
abstractFields = abstract, description, bodytext
contentFields = header, bodytext
}
}
}

View file

@ -19,6 +19,7 @@ plugin {
indexer = Codappix\SearchCore\Domain\Index\TcaIndexer\PagesIndexer
additionalWhereClause = {$plugin.tx_searchcore.settings.indexing.pages.additionalWhereClause}
abstractFields = {$plugin.tx_searchcore.settings.indexing.pages.abstractFields}
contentFields = {$plugin.tx_searchcore.settings.indexing.pages.contentFields}
}
}

View file

@ -5,6 +5,7 @@ Changelog
:maxdepth: 1
:glob:
changelog/20180415-134-make-conent-fields-configurable
changelog/20180409-25-provide-sys-language-uid
changelog/20180408-131-respect-page-cache-clear
changelog/20180408-introduce-php70-type-hints

View file

@ -0,0 +1,13 @@
FEATURE 134 "Enable indexing of tt_content records of CType Header"
===================================================================
Before, only ``bodytext`` was used to generate content while indexing pages.
As there are content elements like ``header`` where this field is empty, but content is still
available, it's now possible to configure the fields.
This makes it also possible to configure further custom content elements with new columns.
A new TypoScript option is now available, and ``header`` is added by default, see
:ref:`contentFields`.
See :issue:`134`.

View file

@ -59,9 +59,9 @@ author = u'Daniel Siepmann'
# built documents.
#
# The short X.Y version.
version = u'0.0.1'
version = u'0.0.3'
# The full version, including alpha/beta/rc tags.
release = u'0.0.1'
release = u'0.0.3'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
@ -133,7 +133,7 @@ html_theme_options = {
# The name for this set of Sphinx documents.
# "<project> v<release> documentation" by default.
#html_title = u'TYPO3 Extension search_core v0.0.1'
#html_title = u'TYPO3 Extension search_core v0.0.3'
# A shorter title for the navigation bar. Default is the same as html_title.
#html_short_title = None

View file

@ -8,6 +8,10 @@ Possible Options:
``to``
Defines the field to copy the values into. All values not false will be copied at the moment.
``from``
Optional, defines the field to copy, can only be one field.
If empty, all existing fields will be copied.
Example::
plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing {
@ -17,7 +21,8 @@ Example::
}
2 = Codappix\SearchCore\DataProcessing\CopyToProcessor
2 {
to = spellcheck
from = uid
to = backup_uid
}
}

View file

@ -86,6 +86,23 @@ Default::
abstract, description, bodytext
.. _contentFields:
contentFields
-------------
Used by: :ref:`PagesIndexer`.
Define which fields should be used to provide the auto generated field "content".
Example::
plugin.tx_searchcore.settings.indexing.pages.contentFields := addToList(table_caption)
Default::
header, bodytext
.. _mapping:
mapping

View file

@ -9,7 +9,7 @@ Composer
The extension can be installed through composer::
composer require "codappix/search_core" "~1.0.0"
composer require "codappix/search_core" "~0.0.3"
Note that you have to allow unstable packages:

View file

@ -41,16 +41,11 @@ unitTests:
.Build/bin/phpunit --colors --debug -v \
-c Tests/Unit/UnitTests.xml
uploadCodeCoverage: uploadCodeCoverageToScrutinizer uploadCodeCoverageToCodacy
uploadCodeCoverage: uploadCodeCoverageToScrutinizer
uploadCodeCoverageToScrutinizer:
wget https://scrutinizer-ci.com/ocular.phar && \
php ocular.phar code-coverage:upload --format=php-clover .Build/report/functional/clover/coverage
uploadCodeCoverageToCodacy:
composer require -vv --dev codacy/coverage && \
git checkout composer.json && \
php .Build/bin/codacycoverage clover .Build/report/functional/clover/coverage
clean:
rm -rf .Build composer.lock

View file

@ -49,12 +49,11 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
$response = $this->client->request('typo3content/_search?q=*:*');
$this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.');
$this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.');
$this->assertArraySubset(
['_source' => ['header' => 'indexed content element']],
$response->getData()['hits']['hits'][1],
false,
$this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.');
$this->assertSame($response->getData()['hits']['total'], 3, 'Not exactly 3 documents were indexed.');
$this->assertSame(
'indexed content element',
$response->getData()['hits']['hits'][2]['_source']['header'],
'Record was not indexed.'
);
}
@ -72,7 +71,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
$response = $this->client->request('typo3content/_search?q=*:*');
$this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.');
$this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.');
$this->assertSame($response->getData()['hits']['total'], 1, 'Not exactly 1 document was indexed.');
$this->assertArraySubset(
['_source' => ['header' => 'indexed content element']],
@ -112,8 +111,8 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
$response = $this->client->request('typo3content/_search?q=*:*');
$this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.');
$this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.');
$this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.');
$this->assertSame($response->getData()['hits']['total'], 3, 'Not exactly 3 documents were indexed.');
}
/**
@ -135,8 +134,8 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
$response = $this->client->request('typo3content/_search?q=*:*');
$this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.');
$this->assertSame($response->getData()['hits']['total'], 3, 'Not exactly 3 documents were indexed.');
$this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.');
$this->assertSame($response->getData()['hits']['total'], 4, 'Not exactly 4 documents were indexed.');
$response = $this->client->request('typo3content/_search?q=uid:11');
$this->assertArraySubset(
['_source' => ['header' => 'Also indexable record']],
@ -167,8 +166,8 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
;
$response = $this->client->request('typo3content/_search?q=*:*');
$this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.');
$this->assertSame($response->getData()['hits']['total'], 4, 'Not exactly 4 documents were indexed.');
$this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.');
$this->assertSame($response->getData()['hits']['total'], 5, 'Not exactly 5 documents were indexed.');
$response = $this->client->request('typo3content/_search?q=uid:11');
$this->assertArraySubset(
@ -209,7 +208,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
/**
* @test
*/
public function indexingDeltedRecordIfRecordShouldBeIndexedButIsNoLongerAvailableAndWasAlreadyIndexed()
public function indexingDeletedRecordIfRecordShouldBeIndexedButIsNoLongerAvailableAndWasAlreadyIndexed()
{
\TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class)
->get(IndexerFactory::class)
@ -218,7 +217,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
;
$response = $this->client->request('typo3content/_search?q=*:*');
$this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.');
$this->assertSame($response->getData()['hits']['total'], 3, 'Not exactly 3 documents were indexed.');
if ($this->isLegacyVersion()) {
$this->getDatabaseConnection()
@ -239,6 +238,6 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase
;
$response = $this->client->request('typo3content/_search?q=*:*');
$this->assertSame($response->getData()['hits']['total'], 1, 'Not exactly 1 document is in index.');
$this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 document is in index.');
}
}

View file

@ -14,7 +14,7 @@ plugin {
additionalWhereClause (
tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu', 'shortcut', 'search', 'login')
AND tt_content.bodytext != ''
AND (tt_content.bodytext != '' OR tt_content.header != '')
)
mapping {
@ -27,6 +27,7 @@ plugin {
pages {
indexer = Codappix\SearchCore\Domain\Index\TcaIndexer\PagesIndexer
abstractFields = abstract, description, bodytext
contentFields = header, bodytext
mapping {
CType {

View file

@ -99,4 +99,31 @@
<colPos>0</colPos>
<filelink_sorting>0</filelink_sorting>
</tt_content>
<tt_content>
<uid>100</uid>
<pid>2</pid>
<tstamp>1480686370</tstamp>
<crdate>1480686370</crdate>
<hidden>0</hidden>
<sorting>72</sorting>
<CType>header</CType>
<header>Indexed on page 2</header>
<bodytext>This element is on a different page</bodytext>
<media>0</media>
<layout>0</layout>
<deleted>0</deleted>
<cols>0</cols>
<starttime>0</starttime>
<endtime>0</endtime>
<colPos>0</colPos>
<filelink_sorting>0</filelink_sorting>
</tt_content>
<pages>
<uid>2</uid>
<pid>1</pid>
<title>Second page with content</title>
<description>Used to check whether content is indexed only for parent page.</description>
</pages>
</dataset>

View file

@ -48,9 +48,11 @@ class PagesIndexerTest extends AbstractFunctionalTestCase
->with(
$this->stringContains($tableName),
$this->callback(function ($documents) {
return count($documents) === 1
return count($documents) === 2
&& isset($documents[0]['content']) && $documents[0]['content'] ===
'this is the content of header content element that should get indexed Some text in paragraph'
'indexed content element' .
' this is the content of header content element that should get indexed' .
' Indexed without html tags Some text in paragraph'
&& isset($documents[0]['search_abstract']) && $documents[0]['search_abstract'] ===
'Used as abstract as no abstract is defined.'
;

View file

@ -80,6 +80,18 @@ class CopyToProcessorTest extends AbstractUnitTestCase
'new_field' => 'Some content like lorem' . PHP_EOL . 'Tag 1' . PHP_EOL . 'Tag 2',
],
],
'Copy single field to new field' => [
'record' => [
'field 1' => 'Some content like lorem',
],
'configuration' => [
'to' => 'new_field',
],
'expectedData' => [
'field 1' => 'Some content like lorem',
'new_field' => 'Some content like lorem',
],
],
];
}
}

View file

@ -36,7 +36,7 @@ class ResultItemTest extends AbstractUnitTestCase
];
$expectedData = $originalData;
$subject = new ResultItem($originalData);
$subject = new ResultItem($originalData, 'testType');
$this->assertSame(
$expectedData,
$subject->getPlainData(),
@ -55,7 +55,7 @@ class ResultItemTest extends AbstractUnitTestCase
];
$expectedData = $originalData;
$subject = new ResultItem($originalData);
$subject = new ResultItem($originalData, 'testType');
$this->assertSame(
$originalData['title'],
$subject['title'],
@ -73,7 +73,7 @@ class ResultItemTest extends AbstractUnitTestCase
'title' => 'Some title',
];
$subject = new ResultItem($originalData);
$subject = new ResultItem($originalData, 'testType');
$this->assertTrue(isset($subject['title']), 'Could not determine that title exists.');
$this->assertFalse(isset($subject['title2']), 'Could not determine that title2 does not exists.');
}
@ -88,7 +88,7 @@ class ResultItemTest extends AbstractUnitTestCase
'title' => 'Some title',
];
$subject = new ResultItem($originalData);
$subject = new ResultItem($originalData, 'testType');
$this->expectException(\BadMethodCallException::class);
$subject['title'] = 'New Title';
}
@ -103,8 +103,57 @@ class ResultItemTest extends AbstractUnitTestCase
'title' => 'Some title',
];
$subject = new ResultItem($originalData);
$subject = new ResultItem($originalData, 'testType');
$this->expectException(\BadMethodCallException::class);
unset($subject['title']);
}
/**
* @test
*/
public function typeCanBeRetrievedAfterConstruction()
{
$originalData = [
'uid' => 10,
'title' => 'Some title',
];
$expectedData = $originalData;
$subject = new ResultItem($originalData, 'testType');
$this->assertSame(
'testType',
$subject->getType(),
'Could not retrieve type.'
);
}
/**
* @test
*/
public function typeCanNotBeChanged()
{
$originalData = [
'uid' => 10,
'title' => 'Some title',
];
$subject = new ResultItem($originalData, 'testType');
$this->expectException(\BadMethodCallException::class);
$subject['type'] = 'New Title';
}
/**
* @test
*/
public function typeCanNotBeRemoved()
{
$originalData = [
'uid' => 10,
'title' => 'Some title',
];
$subject = new ResultItem($originalData, 'testType');
$this->expectException(\BadMethodCallException::class);
unset($subject['type']);
}
}

View file

@ -71,16 +71,25 @@ class SearchResultTest extends AbstractUnitTestCase
$originalSearchResultMock = $this->getMockBuilder(SearchResultInterface::class)->getMock();
$data = [
[
'uid' => 10,
'title' => 'Some Title',
'data' => [
'uid' => 10,
'title' => 'Some Title',
],
'type' => 'testType1',
],
[
'uid' => 11,
'title' => 'Some Title 2',
'data' => [
'uid' => 11,
'title' => 'Some Title 2',
],
'type' => 'testType2',
],
[
'uid' => 12,
'title' => 'Some Title 3',
'data' => [
'uid' => 12,
'title' => 'Some Title 3',
],
'type' => 'testType2',
],
];
@ -89,9 +98,12 @@ class SearchResultTest extends AbstractUnitTestCase
$this->assertCount(3, $resultItems);
$this->assertSame($data[0]['uid'], $resultItems[0]['uid']);
$this->assertSame($data[1]['uid'], $resultItems[1]['uid']);
$this->assertSame($data[2]['uid'], $resultItems[2]['uid']);
$this->assertSame($data[0]['data']['uid'], $resultItems[0]['uid']);
$this->assertSame($data[0]['type'], $resultItems[0]->getType());
$this->assertSame($data[1]['data']['uid'], $resultItems[1]['uid']);
$this->assertSame($data[1]['type'], $resultItems[1]->getType());
$this->assertSame($data[2]['data']['uid'], $resultItems[2]['uid']);
$this->assertSame($data[2]['type'], $resultItems[2]->getType());
$this->assertInstanceOf(ResultItemInterface::class, $resultItems[0]);
$this->assertInstanceOf(ResultItemInterface::class, $resultItems[1]);

View file

@ -294,7 +294,14 @@ class SearchServiceTest extends AbstractUnitTestCase
));
$searchResultMock = $this->getMockBuilder(SearchResultInterface::class)->getMock();
$searchResult = new SearchResult($searchResultMock, [['field 1' => 'value 1']]);
$searchResult = new SearchResult($searchResultMock, [
[
'data' => [
'field 1' => 'value 1'
],
'type' => 'testType',
],
]);
$this->connection->expects($this->once())
->method('search')
@ -311,7 +318,13 @@ class SearchServiceTest extends AbstractUnitTestCase
$this->objectManager->expects($this->once())
->method('get')
->with(SearchResult::class, $searchResult, [
['field 1' => 'value 1', 'field 2' => 'value 2']
[
'data' => [
'field 1' => 'value 1',
'field 2' => 'value 2',
],
'type' => 'testType',
]
])
->willReturn($searchResultMock);

View file

@ -91,4 +91,46 @@ class DataHandlerToProcessorTest extends AbstractUnitTestCase
],
];
}
/**
* @test
*/
public function indexingIsNotCalledForCacheClearIfDataIsInvalid()
{
$coreDataHandlerMock = $this->getMockBuilder(CoreDataHandler::class)->getMock();
$ownDataHandlerMock = $this->getMockBuilder(OwnDataHandler::class)
->disableOriginalConstructor()
->getMock();
$subject = new DataHandler($ownDataHandlerMock);
$ownDataHandlerMock->expects($this->never())->method('update');
$subject->clearCachePostProc([
'cacheCmd' => 'NEW343',
], $coreDataHandlerMock);
}
/**
* @test
*/
public function indexingIsNotCalledForProcessIfDataIsInvalid()
{
$coreDataHandlerMock = $this->getMockBuilder(CoreDataHandler::class)->getMock();
$coreDataHandlerMock->datamap = [
'tt_content' => [
'NEW343' => [],
],
];
$coreDataHandlerMock->substNEWwithIDs = [];
$ownDataHandlerMock = $this->getMockBuilder(OwnDataHandler::class)
->disableOriginalConstructor()
->getMock();
$subject = new DataHandler($ownDataHandlerMock);
$ownDataHandlerMock->expects($this->never())->method('update');
$subject->processDatamap_afterAllOperations($coreDataHandlerMock);
}
}

View file

@ -18,7 +18,7 @@ $EM_CONF[$_EXTKEY] = [
],
],
'state' => 'beta',
'version' => '0.0.1',
'version' => '0.0.3',
'author' => 'Daniel Siepmann',
'author_email' => 'coding@daniel-siepmann.de',
];