WIP|FEATURE: Split tokenizing into dedicated classes

* As each node has to be handled differently.
* Also we can attach many information for some nodes to the token, to
  make linting eaiser. But that's specific per node and those goes to an
  own class.

Relates #67

composer.json

@@ -16,15 +16,18 @@
     },
     "autoload": {
         "psr-4": {
-            "Typo3Update\\": "src/Standards/Typo3Update/"
+            "Typo3Update\\": "src/Standards/Typo3Update/",
+            "Typo3Update\\CodeSniffer\\": "src/CodeSniffer/"
         },
         "files": [
-            "src/CodeSniffer/Tokenizers/TypoScript.php"
+            "src/CodeSniffer/Tokenizers/TypoScript.php",
+            "src/CodeSniffer/Tokenizers/Fluid.php"
         ]
     },
     "require": {
         "php": ">=5.6",
         "helmich/typo3-typoscript-parser": "1.1.*",
+        "typo3fluid/fluid": "2.3.*",
         "squizlabs/php_codesniffer": "2.8.*",
         "symfony/yaml": "3.2.*",
         "higidi/composer-phpcodesniffer-standards-plugin": "*"

src/CodeSniffer/Tokenizers/Fluid.php

@@ -0,0 +1,121 @@
+<?php
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use TYPO3Fluid\Fluid\Core\Parser\ParsingState;
+use TYPO3Fluid\Fluid\Core\Parser\SyntaxTree;
+use TYPO3Fluid\Fluid\Core\Parser\SyntaxTree\NodeInterface;
+use TYPO3Fluid\Fluid\Core\Parser\TemplateParser;
+use TYPO3Fluid\Fluid\Core\Rendering\RenderingContext;
+use TYPO3Fluid\Fluid\View\TemplateView;
+use TYPO3Fluid\Fluid\Core\ViewHelper\ViewHelperResolver;
+use Typo3Update\CodeSniffer\Tokenizers\Fluid\NodeTokenizer;;
+
+class PHP_CodeSniffer_Tokenizers_FLUID
+{
+    /**
+     * If TRUE, files that appear to be minified will not be processed.
+     *
+     * @var boolean
+     */
+    public $skipMinified = false;
+
+    /**
+     * Creates an array of tokens when given some TypoScript code.
+     *
+     * @param string $string  The string to tokenize.
+     * @param string $eolChar The EOL character to use for splitting strings.
+     *
+     * @return array
+     */
+    public function tokenizeString($string, $eolChar = "\n")
+    {
+        $tokens = [];
+        $parser = new TemplateParser();
+        $renderingContext = new RenderingContext(new TemplateView());
+        $viewHelperResolver = new ViewHelperResolver();
+        // TODO: Configure ... somehow? Autodetect?!
+        $viewHelperResolver->addNamespaces([
+            'f' => [
+                'TYPO3Fluid\\Fluid\\ViewHelpers',
+                'TYPO3\CMS\Fluid\ViewHelpers',
+            ],
+            'vendor' => ['something'],
+        ]);
+        $renderingContext->setViewHelperResolver($viewHelperResolver);
+        $parser->setRenderingContext($renderingContext);
+
+        $this->tokenizeNode($parser->parse($string)->getRootNode(), $tokens);
+
+        return $tokens;
+    }
+
+    protected function tokenizeNode(NodeInterface $node, array &$tokens)
+    {
+        if (($node instanceof SyntaxTree\RootNode) === false) {
+            $tokens[] = $this->convertNodeToToken($node);
+        }
+
+        foreach ($node->getChildNodes() as $node) {
+            $this->tokenizeNode($node, $tokens);
+        }
+    }
+
+    protected function convertNodeToToken(NodeInterface $node)
+    {
+        // TODO: How to get line (e.g. closing tag of ViewHelper on another line?!)
+        // TODO: How to get the real content, to allow phpcs determine real length for column?!
+        switch (get_class($node)) {
+            case SyntaxTree\EscapingNode::class:
+                return $this->convertNodeToToken($node->getNode());
+            case SyntaxTree\ArrayNode::class:
+                return 'ArrayNode';
+            case SyntaxTree\BooleanNode::class:
+                return 'BooleanNode';
+            case SyntaxTree\NumericNode::class:
+                return 'NumericNode';
+            case SyntaxTree\ObjectAccessorNode::class:
+                return (new NodeTokenizer\ObjectPathNode($node))->getToken();
+            case SyntaxTree\ViewHelperNode::class:
+                return (new NodeTokenizer\ViewHelperNode($node))->getToken();
+            case SyntaxTree\TextNode::class:
+                return (new NodeTokenizer\TextNode($node))->getToken();
+            default:
+                throw new \Exception('Unknown node.', 1493905030);
+        }
+    }
+
+    /**
+     * Allow the tokenizer to do additional processing if required.
+     *
+     * @param array  $tokens  The array of tokens to process.
+     * @param string $eolChar The EOL character to use for splitting strings.
+     *
+     * @return void
+     *
+     * @SuppressWarnings(PHPMD.UnusedFormalParameter) We need to match the signature.
+     */
+    public function processAdditional(&$tokens, $eolChar)
+    {
+        var_dump($tokens);
+        die;
+        return;
+    }
+}

src/CodeSniffer/Tokenizers/Fluid/NodeTokenizer/AbstractNode.php

@@ -0,0 +1,36 @@
+<?php
+namespace Typo3Update\CodeSniffer\Tokenizers\Fluid\NodeTokenizer;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use TYPO3Fluid\Fluid\Core\Parser\SyntaxTree\NodeInterface;
+
+class AbstractNode
+{
+    /**
+     * @var NodeInterfaceA
+     */
+    protected $node;
+
+    public function __construct(NodeInterface $node)
+    {
+        $this->node = $node;
+    }
+}

src/CodeSniffer/Tokenizers/Fluid/NodeTokenizer/ObjectPathNode.php

@@ -0,0 +1,33 @@
+<?php
+namespace Typo3Update\CodeSniffer\Tokenizers\Fluid\NodeTokenizer;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+class ObjectPathNode extends AbstractNode
+{
+    public function getToken()
+    {
+        return [
+            'code' => 'Text',
+            'type' => 'Text',
+            'content' => '{' . $this->node->getObjectPath() . '}',
+        ];
+    }
+}

src/CodeSniffer/Tokenizers/Fluid/NodeTokenizer/TextNode.php

@@ -0,0 +1,33 @@
+<?php
+namespace Typo3Update\CodeSniffer\Tokenizers\Fluid\NodeTokenizer;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+class TextNode extends AbstractNode
+{
+    public function getToken()
+    {
+        return [
+            'code' => 'Text',
+            'type' => 'Text',
+            'content' => $this->node->getText(),
+        ];
+    }
+}

src/CodeSniffer/Tokenizers/Fluid/NodeTokenizer/ViewHelperNode.php

@@ -0,0 +1,33 @@
+<?php
+namespace Typo3Update\CodeSniffer\Tokenizers\Fluid\NodeTokenizer;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+class ViewHelperNode extends AbstractNode
+{
+    public function getToken()
+    {
+        return [
+            'code' => 'ViewHelper',
+            'type' => 'ViewHelper',
+            'content' => $this->node->getViewHelperClassName(),
+        ];
+    }
+}

tests/CodeSniffer/Tokenizers/FluidTest.php

@@ -0,0 +1,66 @@
+<?php
+
+namespace Typo3Update\Tests\CodeSniffer\Tokenizers;
+
+/*
+ * Copyright (C) 2017  Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use PHPUnit\Framework\TestCase;
+use PHP_CodeSniffer as PhpCs;
+use PHP_CodeSniffer_File as PhpCsFile;
+
+class FluidTest extends TestCase
+{
+    /**
+     * @test
+     */
+    public function callingTokenizerWorksAsExpected()
+    {
+        // Initialize constants, etc.
+        new PhpCs();
+
+        $subject = new \PHP_CodeSniffer_Tokenizers_FLUID();
+        $resultFile = implode(DIRECTORY_SEPARATOR, [
+            __DIR__,
+            '..',
+            '..',
+            'Fixtures',
+            'CodeSniffer',
+            'Tokenizers',
+            'Fluid',
+            'expected.php',
+        ]);
+        $testFile = implode(DIRECTORY_SEPARATOR, [
+            __DIR__,
+            '..',
+            '..',
+            'Fixtures',
+            'CodeSniffer',
+            'Tokenizers',
+            'Fluid',
+            'example.html',
+        ]);
+
+        $this->assertEquals(
+            require $resultFile,
+            PhpCsFile::tokenizeString(file_get_contents($testFile), $subject, "\n"),
+            'Did not get expected tokens.'
+        );
+    }
+}

tests/Fixtures/CodeSniffer/Tokenizers/Fluid/example.html

@@ -0,0 +1,28 @@
+<!-- <hr> -->
+<!-- <f:format.htmlspecialchars> -->
+<!--     <h1>Escaped html</h1> -->
+<!-- </f:format.htmlspecialchars> -->
+<!-- {some.objectOrArray} -->
+
+{namespace vendor = Tx_ExtName_ViewHelpers}
+<f:layout name="Default" />
+<f:section name="SomeSection">
+<f:cObject typoscriptObjectPath="lib.contentGetHead"> </f:cObject>
+    <f:cObject typoscriptObjectPath="lib.breadcrumb"> </f:cObject>
+
+    <div class="container">
+        <div class="row">
+            <aside class="span6">
+                <f:cObject typoscriptObjectPath="lib.sidebar_navigation"> </f:cObject>
+                <f:cObject typoscriptObjectPath="lib.contentGetSidebar"> </f:cObject>
+                <vendor:newsletterBox groupId="96" nlCampaign="3" nlHerkunft="8" additional="" />
+            </aside>
+            <div class="span18">
+                <!--TYPO3SEARCH_begin-->
+                <f:render partial="Content" />
+                <!--TYPO3SEARCH_end-->
+            </div>
+        </div>
+    </div>
+    <f:cObject typoscriptObjectPath="lib.contentGetFoot"> </f:cObject>
+</f:section>

tests/Fixtures/CodeSniffer/Tokenizers/Fluid/expected.php

@@ -0,0 +1,4 @@
+<?php
+
+return [
+];