WIP|FEATURE: Split tokenizing into dedicated classes

* As each node has to be handled differently.
* Also we can attach many information for some nodes to the token, to
  make linting eaiser. But that's specific per node and those goes to an
  own class.

Relates #67
This commit is contained in:
Daniel Siepmann 2017-05-04 15:58:36 +02:00
parent 413d8e45f5
commit af8244f986
Signed by: Daniel Siepmann
GPG key ID: 33D6629915560EF4
9 changed files with 359 additions and 2 deletions

View file

@ -16,15 +16,18 @@
},
"autoload": {
"psr-4": {
"Typo3Update\\": "src/Standards/Typo3Update/"
"Typo3Update\\": "src/Standards/Typo3Update/",
"Typo3Update\\CodeSniffer\\": "src/CodeSniffer/"
},
"files": [
"src/CodeSniffer/Tokenizers/TypoScript.php"
"src/CodeSniffer/Tokenizers/TypoScript.php",
"src/CodeSniffer/Tokenizers/Fluid.php"
]
},
"require": {
"php": ">=5.6",
"helmich/typo3-typoscript-parser": "1.1.*",
"typo3fluid/fluid": "2.3.*",
"squizlabs/php_codesniffer": "2.8.*",
"symfony/yaml": "3.2.*",
"higidi/composer-phpcodesniffer-standards-plugin": "*"

View file

@ -0,0 +1,121 @@
<?php
/*
* Copyright (C) 2017 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
use TYPO3Fluid\Fluid\Core\Parser\ParsingState;
use TYPO3Fluid\Fluid\Core\Parser\SyntaxTree;
use TYPO3Fluid\Fluid\Core\Parser\SyntaxTree\NodeInterface;
use TYPO3Fluid\Fluid\Core\Parser\TemplateParser;
use TYPO3Fluid\Fluid\Core\Rendering\RenderingContext;
use TYPO3Fluid\Fluid\View\TemplateView;
use TYPO3Fluid\Fluid\Core\ViewHelper\ViewHelperResolver;
use Typo3Update\CodeSniffer\Tokenizers\Fluid\NodeTokenizer;;
class PHP_CodeSniffer_Tokenizers_FLUID
{
/**
* If TRUE, files that appear to be minified will not be processed.
*
* @var boolean
*/
public $skipMinified = false;
/**
* Creates an array of tokens when given some TypoScript code.
*
* @param string $string The string to tokenize.
* @param string $eolChar The EOL character to use for splitting strings.
*
* @return array
*/
public function tokenizeString($string, $eolChar = "\n")
{
$tokens = [];
$parser = new TemplateParser();
$renderingContext = new RenderingContext(new TemplateView());
$viewHelperResolver = new ViewHelperResolver();
// TODO: Configure ... somehow? Autodetect?!
$viewHelperResolver->addNamespaces([
'f' => [
'TYPO3Fluid\\Fluid\\ViewHelpers',
'TYPO3\CMS\Fluid\ViewHelpers',
],
'vendor' => ['something'],
]);
$renderingContext->setViewHelperResolver($viewHelperResolver);
$parser->setRenderingContext($renderingContext);
$this->tokenizeNode($parser->parse($string)->getRootNode(), $tokens);
return $tokens;
}
protected function tokenizeNode(NodeInterface $node, array &$tokens)
{
if (($node instanceof SyntaxTree\RootNode) === false) {
$tokens[] = $this->convertNodeToToken($node);
}
foreach ($node->getChildNodes() as $node) {
$this->tokenizeNode($node, $tokens);
}
}
protected function convertNodeToToken(NodeInterface $node)
{
// TODO: How to get line (e.g. closing tag of ViewHelper on another line?!)
// TODO: How to get the real content, to allow phpcs determine real length for column?!
switch (get_class($node)) {
case SyntaxTree\EscapingNode::class:
return $this->convertNodeToToken($node->getNode());
case SyntaxTree\ArrayNode::class:
return 'ArrayNode';
case SyntaxTree\BooleanNode::class:
return 'BooleanNode';
case SyntaxTree\NumericNode::class:
return 'NumericNode';
case SyntaxTree\ObjectAccessorNode::class:
return (new NodeTokenizer\ObjectPathNode($node))->getToken();
case SyntaxTree\ViewHelperNode::class:
return (new NodeTokenizer\ViewHelperNode($node))->getToken();
case SyntaxTree\TextNode::class:
return (new NodeTokenizer\TextNode($node))->getToken();
default:
throw new \Exception('Unknown node.', 1493905030);
}
}
/**
* Allow the tokenizer to do additional processing if required.
*
* @param array $tokens The array of tokens to process.
* @param string $eolChar The EOL character to use for splitting strings.
*
* @return void
*
* @SuppressWarnings(PHPMD.UnusedFormalParameter) We need to match the signature.
*/
public function processAdditional(&$tokens, $eolChar)
{
var_dump($tokens);
die;
return;
}
}

View file

@ -0,0 +1,36 @@
<?php
namespace Typo3Update\CodeSniffer\Tokenizers\Fluid\NodeTokenizer;
/*
* Copyright (C) 2017 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
use TYPO3Fluid\Fluid\Core\Parser\SyntaxTree\NodeInterface;
class AbstractNode
{
/**
* @var NodeInterfaceA
*/
protected $node;
public function __construct(NodeInterface $node)
{
$this->node = $node;
}
}

View file

@ -0,0 +1,33 @@
<?php
namespace Typo3Update\CodeSniffer\Tokenizers\Fluid\NodeTokenizer;
/*
* Copyright (C) 2017 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
class ObjectPathNode extends AbstractNode
{
public function getToken()
{
return [
'code' => 'Text',
'type' => 'Text',
'content' => '{' . $this->node->getObjectPath() . '}',
];
}
}

View file

@ -0,0 +1,33 @@
<?php
namespace Typo3Update\CodeSniffer\Tokenizers\Fluid\NodeTokenizer;
/*
* Copyright (C) 2017 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
class TextNode extends AbstractNode
{
public function getToken()
{
return [
'code' => 'Text',
'type' => 'Text',
'content' => $this->node->getText(),
];
}
}

View file

@ -0,0 +1,33 @@
<?php
namespace Typo3Update\CodeSniffer\Tokenizers\Fluid\NodeTokenizer;
/*
* Copyright (C) 2017 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
class ViewHelperNode extends AbstractNode
{
public function getToken()
{
return [
'code' => 'ViewHelper',
'type' => 'ViewHelper',
'content' => $this->node->getViewHelperClassName(),
];
}
}

View file

@ -0,0 +1,66 @@
<?php
namespace Typo3Update\Tests\CodeSniffer\Tokenizers;
/*
* Copyright (C) 2017 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
use PHPUnit\Framework\TestCase;
use PHP_CodeSniffer as PhpCs;
use PHP_CodeSniffer_File as PhpCsFile;
class FluidTest extends TestCase
{
/**
* @test
*/
public function callingTokenizerWorksAsExpected()
{
// Initialize constants, etc.
new PhpCs();
$subject = new \PHP_CodeSniffer_Tokenizers_FLUID();
$resultFile = implode(DIRECTORY_SEPARATOR, [
__DIR__,
'..',
'..',
'Fixtures',
'CodeSniffer',
'Tokenizers',
'Fluid',
'expected.php',
]);
$testFile = implode(DIRECTORY_SEPARATOR, [
__DIR__,
'..',
'..',
'Fixtures',
'CodeSniffer',
'Tokenizers',
'Fluid',
'example.html',
]);
$this->assertEquals(
require $resultFile,
PhpCsFile::tokenizeString(file_get_contents($testFile), $subject, "\n"),
'Did not get expected tokens.'
);
}
}

View file

@ -0,0 +1,28 @@
<!-- <hr> -->
<!-- <f:format.htmlspecialchars> -->
<!-- <h1>Escaped html</h1> -->
<!-- </f:format.htmlspecialchars> -->
<!-- {some.objectOrArray} -->
{namespace vendor = Tx_ExtName_ViewHelpers}
<f:layout name="Default" />
<f:section name="SomeSection">
<f:cObject typoscriptObjectPath="lib.contentGetHead"> </f:cObject>
<f:cObject typoscriptObjectPath="lib.breadcrumb"> </f:cObject>
<div class="container">
<div class="row">
<aside class="span6">
<f:cObject typoscriptObjectPath="lib.sidebar_navigation"> </f:cObject>
<f:cObject typoscriptObjectPath="lib.contentGetSidebar"> </f:cObject>
<vendor:newsletterBox groupId="96" nlCampaign="3" nlHerkunft="8" additional="" />
</aside>
<div class="span18">
<!--TYPO3SEARCH_begin-->
<f:render partial="Content" />
<!--TYPO3SEARCH_end-->
</div>
</div>
</div>
<f:cObject typoscriptObjectPath="lib.contentGetFoot"> </f:cObject>
</f:section>

View file

@ -0,0 +1,4 @@
<?php
return [
];