From af8244f9868c20981fa9847ce27dbc4110733663 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 4 May 2017 15:58:36 +0200 Subject: [PATCH] WIP|FEATURE: Split tokenizing into dedicated classes * As each node has to be handled differently. * Also we can attach many information for some nodes to the token, to make linting eaiser. But that's specific per node and those goes to an own class. Relates #67 --- composer.json | 7 +- src/CodeSniffer/Tokenizers/Fluid.php | 121 ++++++++++++++++++ .../Fluid/NodeTokenizer/AbstractNode.php | 36 ++++++ .../Fluid/NodeTokenizer/ObjectPathNode.php | 33 +++++ .../Fluid/NodeTokenizer/TextNode.php | 33 +++++ .../Fluid/NodeTokenizer/ViewHelperNode.php | 33 +++++ tests/CodeSniffer/Tokenizers/FluidTest.php | 66 ++++++++++ .../CodeSniffer/Tokenizers/Fluid/example.html | 28 ++++ .../CodeSniffer/Tokenizers/Fluid/expected.php | 4 + 9 files changed, 359 insertions(+), 2 deletions(-) create mode 100644 src/CodeSniffer/Tokenizers/Fluid.php create mode 100644 src/CodeSniffer/Tokenizers/Fluid/NodeTokenizer/AbstractNode.php create mode 100644 src/CodeSniffer/Tokenizers/Fluid/NodeTokenizer/ObjectPathNode.php create mode 100644 src/CodeSniffer/Tokenizers/Fluid/NodeTokenizer/TextNode.php create mode 100644 src/CodeSniffer/Tokenizers/Fluid/NodeTokenizer/ViewHelperNode.php create mode 100644 tests/CodeSniffer/Tokenizers/FluidTest.php create mode 100644 tests/Fixtures/CodeSniffer/Tokenizers/Fluid/example.html create mode 100644 tests/Fixtures/CodeSniffer/Tokenizers/Fluid/expected.php diff --git a/composer.json b/composer.json index 623bfde..7ba2fa4 100644 --- a/composer.json +++ b/composer.json @@ -16,15 +16,18 @@ }, "autoload": { "psr-4": { - "Typo3Update\\": "src/Standards/Typo3Update/" + "Typo3Update\\": "src/Standards/Typo3Update/", + "Typo3Update\\CodeSniffer\\": "src/CodeSniffer/" }, "files": [ - "src/CodeSniffer/Tokenizers/TypoScript.php" + "src/CodeSniffer/Tokenizers/TypoScript.php", + "src/CodeSniffer/Tokenizers/Fluid.php" ] }, "require": { "php": ">=5.6", "helmich/typo3-typoscript-parser": "1.1.*", + "typo3fluid/fluid": "2.3.*", "squizlabs/php_codesniffer": "2.8.*", "symfony/yaml": "3.2.*", "higidi/composer-phpcodesniffer-standards-plugin": "*" diff --git a/src/CodeSniffer/Tokenizers/Fluid.php b/src/CodeSniffer/Tokenizers/Fluid.php new file mode 100644 index 0000000..5fa0dc0 --- /dev/null +++ b/src/CodeSniffer/Tokenizers/Fluid.php @@ -0,0 +1,121 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use TYPO3Fluid\Fluid\Core\Parser\ParsingState; +use TYPO3Fluid\Fluid\Core\Parser\SyntaxTree; +use TYPO3Fluid\Fluid\Core\Parser\SyntaxTree\NodeInterface; +use TYPO3Fluid\Fluid\Core\Parser\TemplateParser; +use TYPO3Fluid\Fluid\Core\Rendering\RenderingContext; +use TYPO3Fluid\Fluid\View\TemplateView; +use TYPO3Fluid\Fluid\Core\ViewHelper\ViewHelperResolver; +use Typo3Update\CodeSniffer\Tokenizers\Fluid\NodeTokenizer;; + +class PHP_CodeSniffer_Tokenizers_FLUID +{ + /** + * If TRUE, files that appear to be minified will not be processed. + * + * @var boolean + */ + public $skipMinified = false; + + /** + * Creates an array of tokens when given some TypoScript code. + * + * @param string $string The string to tokenize. + * @param string $eolChar The EOL character to use for splitting strings. + * + * @return array + */ + public function tokenizeString($string, $eolChar = "\n") + { + $tokens = []; + $parser = new TemplateParser(); + $renderingContext = new RenderingContext(new TemplateView()); + $viewHelperResolver = new ViewHelperResolver(); + // TODO: Configure ... somehow? Autodetect?! + $viewHelperResolver->addNamespaces([ + 'f' => [ + 'TYPO3Fluid\\Fluid\\ViewHelpers', + 'TYPO3\CMS\Fluid\ViewHelpers', + ], + 'vendor' => ['something'], + ]); + $renderingContext->setViewHelperResolver($viewHelperResolver); + $parser->setRenderingContext($renderingContext); + + $this->tokenizeNode($parser->parse($string)->getRootNode(), $tokens); + + return $tokens; + } + + protected function tokenizeNode(NodeInterface $node, array &$tokens) + { + if (($node instanceof SyntaxTree\RootNode) === false) { + $tokens[] = $this->convertNodeToToken($node); + } + + foreach ($node->getChildNodes() as $node) { + $this->tokenizeNode($node, $tokens); + } + } + + protected function convertNodeToToken(NodeInterface $node) + { + // TODO: How to get line (e.g. closing tag of ViewHelper on another line?!) + // TODO: How to get the real content, to allow phpcs determine real length for column?! + switch (get_class($node)) { + case SyntaxTree\EscapingNode::class: + return $this->convertNodeToToken($node->getNode()); + case SyntaxTree\ArrayNode::class: + return 'ArrayNode'; + case SyntaxTree\BooleanNode::class: + return 'BooleanNode'; + case SyntaxTree\NumericNode::class: + return 'NumericNode'; + case SyntaxTree\ObjectAccessorNode::class: + return (new NodeTokenizer\ObjectPathNode($node))->getToken(); + case SyntaxTree\ViewHelperNode::class: + return (new NodeTokenizer\ViewHelperNode($node))->getToken(); + case SyntaxTree\TextNode::class: + return (new NodeTokenizer\TextNode($node))->getToken(); + default: + throw new \Exception('Unknown node.', 1493905030); + } + } + + /** + * Allow the tokenizer to do additional processing if required. + * + * @param array $tokens The array of tokens to process. + * @param string $eolChar The EOL character to use for splitting strings. + * + * @return void + * + * @SuppressWarnings(PHPMD.UnusedFormalParameter) We need to match the signature. + */ + public function processAdditional(&$tokens, $eolChar) + { + var_dump($tokens); + die; + return; + } +} diff --git a/src/CodeSniffer/Tokenizers/Fluid/NodeTokenizer/AbstractNode.php b/src/CodeSniffer/Tokenizers/Fluid/NodeTokenizer/AbstractNode.php new file mode 100644 index 0000000..583996e --- /dev/null +++ b/src/CodeSniffer/Tokenizers/Fluid/NodeTokenizer/AbstractNode.php @@ -0,0 +1,36 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use TYPO3Fluid\Fluid\Core\Parser\SyntaxTree\NodeInterface; + +class AbstractNode +{ + /** + * @var NodeInterfaceA + */ + protected $node; + + public function __construct(NodeInterface $node) + { + $this->node = $node; + } +} diff --git a/src/CodeSniffer/Tokenizers/Fluid/NodeTokenizer/ObjectPathNode.php b/src/CodeSniffer/Tokenizers/Fluid/NodeTokenizer/ObjectPathNode.php new file mode 100644 index 0000000..5bc0c81 --- /dev/null +++ b/src/CodeSniffer/Tokenizers/Fluid/NodeTokenizer/ObjectPathNode.php @@ -0,0 +1,33 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +class ObjectPathNode extends AbstractNode +{ + public function getToken() + { + return [ + 'code' => 'Text', + 'type' => 'Text', + 'content' => '{' . $this->node->getObjectPath() . '}', + ]; + } +} diff --git a/src/CodeSniffer/Tokenizers/Fluid/NodeTokenizer/TextNode.php b/src/CodeSniffer/Tokenizers/Fluid/NodeTokenizer/TextNode.php new file mode 100644 index 0000000..338a8ad --- /dev/null +++ b/src/CodeSniffer/Tokenizers/Fluid/NodeTokenizer/TextNode.php @@ -0,0 +1,33 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +class TextNode extends AbstractNode +{ + public function getToken() + { + return [ + 'code' => 'Text', + 'type' => 'Text', + 'content' => $this->node->getText(), + ]; + } +} diff --git a/src/CodeSniffer/Tokenizers/Fluid/NodeTokenizer/ViewHelperNode.php b/src/CodeSniffer/Tokenizers/Fluid/NodeTokenizer/ViewHelperNode.php new file mode 100644 index 0000000..3b7e3fe --- /dev/null +++ b/src/CodeSniffer/Tokenizers/Fluid/NodeTokenizer/ViewHelperNode.php @@ -0,0 +1,33 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +class ViewHelperNode extends AbstractNode +{ + public function getToken() + { + return [ + 'code' => 'ViewHelper', + 'type' => 'ViewHelper', + 'content' => $this->node->getViewHelperClassName(), + ]; + } +} diff --git a/tests/CodeSniffer/Tokenizers/FluidTest.php b/tests/CodeSniffer/Tokenizers/FluidTest.php new file mode 100644 index 0000000..db0fc1b --- /dev/null +++ b/tests/CodeSniffer/Tokenizers/FluidTest.php @@ -0,0 +1,66 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use PHPUnit\Framework\TestCase; +use PHP_CodeSniffer as PhpCs; +use PHP_CodeSniffer_File as PhpCsFile; + +class FluidTest extends TestCase +{ + /** + * @test + */ + public function callingTokenizerWorksAsExpected() + { + // Initialize constants, etc. + new PhpCs(); + + $subject = new \PHP_CodeSniffer_Tokenizers_FLUID(); + $resultFile = implode(DIRECTORY_SEPARATOR, [ + __DIR__, + '..', + '..', + 'Fixtures', + 'CodeSniffer', + 'Tokenizers', + 'Fluid', + 'expected.php', + ]); + $testFile = implode(DIRECTORY_SEPARATOR, [ + __DIR__, + '..', + '..', + 'Fixtures', + 'CodeSniffer', + 'Tokenizers', + 'Fluid', + 'example.html', + ]); + + $this->assertEquals( + require $resultFile, + PhpCsFile::tokenizeString(file_get_contents($testFile), $subject, "\n"), + 'Did not get expected tokens.' + ); + } +} diff --git a/tests/Fixtures/CodeSniffer/Tokenizers/Fluid/example.html b/tests/Fixtures/CodeSniffer/Tokenizers/Fluid/example.html new file mode 100644 index 0000000..0329ae3 --- /dev/null +++ b/tests/Fixtures/CodeSniffer/Tokenizers/Fluid/example.html @@ -0,0 +1,28 @@ + + + + + + +{namespace vendor = Tx_ExtName_ViewHelpers} + + + + + +
+
+ +
+ + + +
+
+
+ + diff --git a/tests/Fixtures/CodeSniffer/Tokenizers/Fluid/expected.php b/tests/Fixtures/CodeSniffer/Tokenizers/Fluid/expected.php new file mode 100644 index 0000000..b625128 --- /dev/null +++ b/tests/Fixtures/CodeSniffer/Tokenizers/Fluid/expected.php @@ -0,0 +1,4 @@ +