From 015931518380b2b3b7a429a3b99ddcd453f75f61 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 8 Nov 2017 20:20:37 +0100 Subject: [PATCH 1/3] FEATURE: Add data processor to remove fields for indexing Add a new processor, with docs and tests, to allow removal of fields before sending them to search service like elasticsearch. E.g. remove sensitive information that should not be available. --- Classes/DataProcessing/RemoveProcessor.php | 44 +++++++ .../dataProcessing/RemoveProcessor.rst | 23 ++++ .../source/configuration/indexing.rst | 1 + .../DataProcessing/RemoveProcessorTest.php | 122 ++++++++++++++++++ 4 files changed, 190 insertions(+) create mode 100644 Classes/DataProcessing/RemoveProcessor.php create mode 100644 Documentation/source/configuration/dataProcessing/RemoveProcessor.rst create mode 100644 Tests/Unit/DataProcessing/RemoveProcessorTest.php diff --git a/Classes/DataProcessing/RemoveProcessor.php b/Classes/DataProcessing/RemoveProcessor.php new file mode 100644 index 0000000..b5ab788 --- /dev/null +++ b/Classes/DataProcessing/RemoveProcessor.php @@ -0,0 +1,44 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use TYPO3\CMS\Core\Utility\GeneralUtility; + +/** + * Removes fields from record. + */ +class RemoveProcessor implements ProcessorInterface +{ + public function processRecord(array $record, array $configuration) : array + { + if (!isset($configuration['fields'])) { + return $record; + } + + foreach (GeneralUtility::trimExplode(',', $configuration['fields'], true) as $field) { + if (isset($record[$field])) { + unset($record[$field]); + } + } + + return $record; + } +} diff --git a/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst b/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst new file mode 100644 index 0000000..197f3c1 --- /dev/null +++ b/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst @@ -0,0 +1,23 @@ +``Codappix\SearchCore\DataProcessing\RemoveProcessor`` +====================================================== + +Will remove fields from record, e.g. if you do not want to sent them to elasticsearch at all. + +Possible Options: + +``fields`` + Comma separated list of fields to remove from record. + +Example:: + + plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing { + 1 = Codappix\SearchCore\DataProcessing\RemoveProcessor + 1 { + fields = description + } + 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 2 { + fields = description, another_field + } + } + diff --git a/Documentation/source/configuration/indexing.rst b/Documentation/source/configuration/indexing.rst index 93eff19..b605008 100644 --- a/Documentation/source/configuration/indexing.rst +++ b/Documentation/source/configuration/indexing.rst @@ -180,6 +180,7 @@ The following Processor are available: :glob: dataProcessing/CopyToProcessor + dataProcessing/RemoveProcessor dataProcessing/GeoPointProcessor The following Processor are planned: diff --git a/Tests/Unit/DataProcessing/RemoveProcessorTest.php b/Tests/Unit/DataProcessing/RemoveProcessorTest.php new file mode 100644 index 0000000..57600f0 --- /dev/null +++ b/Tests/Unit/DataProcessing/RemoveProcessorTest.php @@ -0,0 +1,122 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\DataProcessing\RemoveProcessor; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class RemoveProcessorTest extends AbstractUnitTestCase +{ + /** + * @test + * @dataProvider getPossibleRecordConfigurationCombinations + */ + public function fieldsAreCopiedAsConfigured(array $record, array $configuration, array $expectedRecord) + { + $subject = new RemoveProcessor(); + $processedRecord = $subject->processRecord($record, $configuration); + $this->assertSame( + $expectedRecord, + $processedRecord, + 'The processor did not return the expected processed record.' + ); + } + + /** + * @return array + */ + public function getPossibleRecordConfigurationCombinations() + { + return [ + 'Nothing configured' => [ + 'record' => [ + 'field 1' => 'Some content like lorem', + 'field with sub2' => [ + 'Tag 1', + 'Tag 2', + ], + ], + 'configuration' => [ + ], + 'expectedRecord' => [ + 'field 1' => 'Some content like lorem', + 'field with sub2' => [ + 'Tag 1', + 'Tag 2', + ], + ], + ], + 'Single field configured' => [ + 'record' => [ + 'field 1' => 'Some content like lorem', + 'field with sub2' => [ + 'Tag 1', + 'Tag 2', + ], + ], + 'configuration' => [ + 'fields' => 'field with sub2', + '_typoScriptNodeValue' => 'Codappix\SearchCore\DataProcessing\RemoveProcessor', + ], + 'expectedRecord' => [ + 'field 1' => 'Some content like lorem', + ], + ], + 'Non existing field configured' => [ + 'record' => [ + 'field 1' => 'Some content like lorem', + 'field with sub2' => [ + 'Tag 1', + 'Tag 2', + ], + ], + 'configuration' => [ + 'fields' => 'non existing', + '_typoScriptNodeValue' => 'Codappix\SearchCore\DataProcessing\RemoveProcessor', + ], + 'expectedRecord' => [ + 'field 1' => 'Some content like lorem', + 'field with sub2' => [ + 'Tag 1', + 'Tag 2', + ], + ], + ], + 'Multiple fields configured' => [ + 'record' => [ + 'field 1' => 'Some content like lorem', + 'field with sub2' => [ + 'Tag 1', + 'Tag 2', + ], + 'field 3' => 'Some more like lorem', + ], + 'configuration' => [ + 'fields' => 'field 3, field with sub2', + '_typoScriptNodeValue' => 'Codappix\SearchCore\DataProcessing\RemoveProcessor', + ], + 'expectedRecord' => [ + 'field 1' => 'Some content like lorem', + ], + ], + ]; + } +} From 379dddf8ac6f7e4bcb526f9f6627524398ecd5ff Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 8 Nov 2017 20:36:04 +0100 Subject: [PATCH 2/3] BUGFIX: Also remove fields containing "null" --- Classes/DataProcessing/RemoveProcessor.php | 2 +- Tests/Unit/DataProcessing/RemoveProcessorTest.php | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/Classes/DataProcessing/RemoveProcessor.php b/Classes/DataProcessing/RemoveProcessor.php index b5ab788..6e74237 100644 --- a/Classes/DataProcessing/RemoveProcessor.php +++ b/Classes/DataProcessing/RemoveProcessor.php @@ -34,7 +34,7 @@ class RemoveProcessor implements ProcessorInterface } foreach (GeneralUtility::trimExplode(',', $configuration['fields'], true) as $field) { - if (isset($record[$field])) { + if (array_key_exists($field, $record)) { unset($record[$field]); } } diff --git a/Tests/Unit/DataProcessing/RemoveProcessorTest.php b/Tests/Unit/DataProcessing/RemoveProcessorTest.php index 57600f0..dc55f73 100644 --- a/Tests/Unit/DataProcessing/RemoveProcessorTest.php +++ b/Tests/Unit/DataProcessing/RemoveProcessorTest.php @@ -117,6 +117,17 @@ class RemoveProcessorTest extends AbstractUnitTestCase 'field 1' => 'Some content like lorem', ], ], + 'Fields with "null" san be removed' => [ + 'record' => [ + 'field 1' => null, + ], + 'configuration' => [ + 'fields' => 'field 1', + '_typoScriptNodeValue' => 'Codappix\SearchCore\DataProcessing\RemoveProcessor', + ], + 'expectedRecord' => [ + ], + ], ]; } } From ddb95e8c9111bcec5679b130af4b2db8c5a7a24b Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 8 Nov 2017 20:37:28 +0100 Subject: [PATCH 3/3] BUGFIX: Fix documentation for remove processor --- .../source/configuration/dataProcessing/RemoveProcessor.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst b/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst index 197f3c1..c8653a5 100644 --- a/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst +++ b/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst @@ -15,7 +15,7 @@ Example:: 1 { fields = description } - 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 2 = Codappix\SearchCore\DataProcessing\RemoveProcessor 2 { fields = description, another_field }