Compare commits
No commits in common. "feature/kickstart" and "master" have entirely different histories.
feature/ki
...
master
11 changed files with 0 additions and 1043 deletions
3
.gitignore
vendored
3
.gitignore
vendored
|
|
@ -1,3 +0,0 @@
|
||||||
/vendor/
|
|
||||||
/composer.lock
|
|
||||||
/output/
|
|
||||||
42
comparison
42
comparison
|
|
@ -1,42 +0,0 @@
|
||||||
#!/usr/bin/env php
|
|
||||||
<?php
|
|
||||||
|
|
||||||
declare(ticks = 1);
|
|
||||||
|
|
||||||
pcntl_signal(SIGINT, function () {
|
|
||||||
exit(99);
|
|
||||||
});
|
|
||||||
|
|
||||||
require __DIR__ . '/vendor/autoload.php';
|
|
||||||
|
|
||||||
use Codappix\WebsiteComparison\Command\CompareCommand;
|
|
||||||
use Codappix\WebsiteComparison\Command\CreateBaseCommand;
|
|
||||||
use Facebook\WebDriver\Chrome\ChromeDriver;
|
|
||||||
use Facebook\WebDriver\Chrome\ChromeDriverService;
|
|
||||||
use Symfony\Component\Console\Application;
|
|
||||||
use Symfony\Component\EventDispatcher\EventDispatcher;
|
|
||||||
|
|
||||||
$eventDispatcher = new EventDispatcher();
|
|
||||||
|
|
||||||
$chromeDriver = (function () {
|
|
||||||
$chromeDriverService = new ChromeDriverService(
|
|
||||||
'/usr/lib/chromium-browser/chromedriver',
|
|
||||||
9515,
|
|
||||||
[
|
|
||||||
'--port=9515',
|
|
||||||
'--headless',
|
|
||||||
]
|
|
||||||
);
|
|
||||||
|
|
||||||
return ChromeDriver::start(null, $chromeDriverService);
|
|
||||||
})();
|
|
||||||
|
|
||||||
|
|
||||||
$application = new Application();
|
|
||||||
$application->setDispatcher($eventDispatcher);
|
|
||||||
|
|
||||||
// TODO: Use factory for commands, which injects event dispatcher and chrome driver?
|
|
||||||
$application->add(new CreateBaseCommand($eventDispatcher, $chromeDriver));
|
|
||||||
$application->add(new CompareCommand($eventDispatcher, $chromeDriver));
|
|
||||||
|
|
||||||
$application->run();
|
|
||||||
|
|
@ -1,26 +0,0 @@
|
||||||
{
|
|
||||||
"name": "codappix/website-comparison",
|
|
||||||
"description": "Compares a Website visually by comparing Screenshots.",
|
|
||||||
"type": "project",
|
|
||||||
"license": "GPL-2.0-or-later",
|
|
||||||
"authors": [
|
|
||||||
{
|
|
||||||
"name": "Daniel Siepmann",
|
|
||||||
"email": "coding@daniel-siepmann.de"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"autoload": {
|
|
||||||
"psr-4": {
|
|
||||||
"Codappix\\WebsiteComparison\\": "src/"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"require": {
|
|
||||||
"php": "^7.2",
|
|
||||||
"ext-imagick": "*",
|
|
||||||
"facebook/webdriver": "^1.6",
|
|
||||||
"symfony/console": "^4.1",
|
|
||||||
"symfony/process": "^4.1",
|
|
||||||
"guzzlehttp/psr7": "^1.4",
|
|
||||||
"symfony/event-dispatcher": "^4.1"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
26
readme.rst
26
readme.rst
|
|
@ -1,26 +0,0 @@
|
||||||
Website Comparison
|
|
||||||
==================
|
|
||||||
|
|
||||||
A small command line tool that will screenshot a whole website. On the next run
|
|
||||||
there can be a comparison to the base that was created before.
|
|
||||||
|
|
||||||
Installation
|
|
||||||
------------
|
|
||||||
|
|
||||||
Run ``composer install``.
|
|
||||||
|
|
||||||
Also ``chromedriver`` is needed. Right now the pat his hardcoded to
|
|
||||||
``/usr/lib/chromium-browser/chromedriver`` in file ``comparison``.
|
|
||||||
|
|
||||||
Also the ``chromium-browser`` binary has to be inside the ``$PATH``.
|
|
||||||
|
|
||||||
Usage
|
|
||||||
-----
|
|
||||||
|
|
||||||
First run ``./comparison comparison:createbase`` to create the "base" version of the
|
|
||||||
website. This way we have the base to compare against later.
|
|
||||||
|
|
||||||
Second run ``./comparison comparison:comparetobase`` to screenshot website again and
|
|
||||||
inform about differences.
|
|
||||||
|
|
||||||
For further possible options add ``--help``.
|
|
||||||
|
|
@ -1,222 +0,0 @@
|
||||||
<?php
|
|
||||||
|
|
||||||
namespace Codappix\WebsiteComparison\Command;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Copyright (C) 2018 Daniel Siepmann <coding@daniel-siepmann.de>
|
|
||||||
*
|
|
||||||
* This program is free software; you can redistribute it and/or
|
|
||||||
* modify it under the terms of the GNU General Public License
|
|
||||||
* as published by the Free Software Foundation; either version 2
|
|
||||||
* of the License, or (at your option) any later version.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
* GNU General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU General Public License
|
|
||||||
* along with this program; if not, write to the Free Software
|
|
||||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
|
||||||
* 02110-1301, USA.
|
|
||||||
*/
|
|
||||||
|
|
||||||
use Codappix\WebsiteComparison\Model\UrlListDto;
|
|
||||||
use Codappix\WebsiteComparison\Model\UrlListDtoFactory;
|
|
||||||
use Codappix\WebsiteComparison\Service\Screenshot\CompareService;
|
|
||||||
use Codappix\WebsiteComparison\Service\Screenshot\CrawlerService;
|
|
||||||
use Codappix\WebsiteComparison\Service\Screenshot\Service;
|
|
||||||
use Facebook\WebDriver\Remote\RemoteWebDriver;
|
|
||||||
use Symfony\Component\Console\Command\Command;
|
|
||||||
use Symfony\Component\Console\Input\InputArgument;
|
|
||||||
use Symfony\Component\Console\Input\InputInterface;
|
|
||||||
use Symfony\Component\Console\Input\InputOption;
|
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
|
||||||
use Symfony\Component\EventDispatcher\EventDispatcherInterface;
|
|
||||||
use Symfony\Component\EventDispatcher\GenericEvent;
|
|
||||||
|
|
||||||
class CompareCommand extends Command
|
|
||||||
{
|
|
||||||
/**
|
|
||||||
* @var EventDispatcherInterface
|
|
||||||
*/
|
|
||||||
protected $eventDispatcher;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @var RemoteWebDriver
|
|
||||||
*/
|
|
||||||
protected $webDriver;
|
|
||||||
|
|
||||||
public function __construct(
|
|
||||||
EventDispatcherInterface $eventDispatcher,
|
|
||||||
RemoteWebDriver $webDriver
|
|
||||||
) {
|
|
||||||
parent::__construct(null);
|
|
||||||
|
|
||||||
$this->eventDispatcher = $eventDispatcher;
|
|
||||||
$this->webDriver = $webDriver;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function configure()
|
|
||||||
{
|
|
||||||
$this
|
|
||||||
->setName('comparison:comparetobase')
|
|
||||||
->setDescription('Compare curent state against saved base.')
|
|
||||||
->setHelp('Crawls and screenshots the original website, as a base for future comparison.')
|
|
||||||
|
|
||||||
->addOption(
|
|
||||||
'screenshotDir',
|
|
||||||
null,
|
|
||||||
InputOption::VALUE_OPTIONAL,
|
|
||||||
'Define the sub directory containing original Screenshots for comparison.',
|
|
||||||
'output/base'
|
|
||||||
)
|
|
||||||
->addOption(
|
|
||||||
'compareDir',
|
|
||||||
null,
|
|
||||||
InputOption::VALUE_OPTIONAL,
|
|
||||||
'Define the sub directory to use for storing created Screenshots.',
|
|
||||||
'output/compare'
|
|
||||||
)
|
|
||||||
->addOption(
|
|
||||||
'diffResultDir',
|
|
||||||
null,
|
|
||||||
InputOption::VALUE_OPTIONAL,
|
|
||||||
'Define the sub directory to use for storing created diffs.',
|
|
||||||
'output/diffResult'
|
|
||||||
)
|
|
||||||
->addOption(
|
|
||||||
'screenshotWidth',
|
|
||||||
null,
|
|
||||||
InputOption::VALUE_OPTIONAL,
|
|
||||||
'The width for screen resolution and screenshots.',
|
|
||||||
3840
|
|
||||||
)
|
|
||||||
->addOption(
|
|
||||||
'recoverFile',
|
|
||||||
null,
|
|
||||||
InputOption::VALUE_OPTIONAL,
|
|
||||||
'Path to json-File with state of stopped process, used to recover process.',
|
|
||||||
''
|
|
||||||
)
|
|
||||||
|
|
||||||
->addArgument(
|
|
||||||
'baseUrl',
|
|
||||||
InputArgument::REQUIRED,
|
|
||||||
'E.g. https://typo3.org/ the base url of the website to crawl.'
|
|
||||||
)
|
|
||||||
;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function execute(InputInterface $input, OutputInterface $output)
|
|
||||||
{
|
|
||||||
$baseUrl = $input->getArgument('baseUrl');
|
|
||||||
|
|
||||||
$screenshotService = new Service(
|
|
||||||
$this->eventDispatcher,
|
|
||||||
$input->getOption('compareDir'),
|
|
||||||
$input->getOption('screenshotWidth')
|
|
||||||
);
|
|
||||||
|
|
||||||
$screenshotCrawler = new CrawlerService(
|
|
||||||
$this->webDriver,
|
|
||||||
$screenshotService,
|
|
||||||
$baseUrl
|
|
||||||
);
|
|
||||||
|
|
||||||
$compareService = new CompareService(
|
|
||||||
$this->eventDispatcher,
|
|
||||||
$screenshotService,
|
|
||||||
$input->getOption('screenshotDir'),
|
|
||||||
$input->getOption('diffResultDir')
|
|
||||||
);
|
|
||||||
|
|
||||||
$linkList = $this->getLinkList($baseUrl, $input->getOption('recoverFile'));
|
|
||||||
|
|
||||||
$this->registerEvents($output, $compareService);
|
|
||||||
try {
|
|
||||||
$screenshotCrawler->crawl($linkList);
|
|
||||||
} catch (\Exception $e) {
|
|
||||||
file_put_contents($this->getJsonFilePath($screenshotService, $baseUrl), json_encode($linkList));
|
|
||||||
$output->writeln(sprintf(
|
|
||||||
'<comment>Saved current state for recovering in "%s".</comment>',
|
|
||||||
$this->getJsonFilePath($screenshotService, $baseUrl)
|
|
||||||
));
|
|
||||||
throw $e;
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($compareService->hasDifferences()) {
|
|
||||||
return 255;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function getLinkList(
|
|
||||||
string $baseUrl,
|
|
||||||
string $recoverFile = ''
|
|
||||||
): UrlListDto {
|
|
||||||
$factory = new UrlListDtoFactory();
|
|
||||||
|
|
||||||
if (trim($recoverFile) !== '') {
|
|
||||||
return $factory->createWithByConfigurationFile($recoverFile);
|
|
||||||
}
|
|
||||||
|
|
||||||
return $factory->createWithBaseUrl($baseUrl);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function getJsonFilePath(Service $screenshotService, string $baseUrl): string
|
|
||||||
{
|
|
||||||
return $screenshotService->getScreenshotDir() .
|
|
||||||
DIRECTORY_SEPARATOR .
|
|
||||||
$screenshotService->getScreenshotTarget($baseUrl, 'json')
|
|
||||||
;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function registerEvents(OutputInterface $output, CompareService $compareService)
|
|
||||||
{
|
|
||||||
$this->eventDispatcher->addListener(
|
|
||||||
'service.screenshot.created',
|
|
||||||
function (GenericEvent $event) use ($output, $compareService) {
|
|
||||||
$output->writeln(sprintf(
|
|
||||||
'<info>Comparing Screenshot for url "%s".</info>',
|
|
||||||
$event->getArgument('url')
|
|
||||||
));
|
|
||||||
$compareService->compareScreenshot(
|
|
||||||
$event->getArgument('screenshot')
|
|
||||||
);
|
|
||||||
}
|
|
||||||
);
|
|
||||||
|
|
||||||
if ($output->isVerbose()) {
|
|
||||||
$this->eventDispatcher->addListener(
|
|
||||||
'service.screenshot.isSame',
|
|
||||||
function (GenericEvent $event) use ($output) {
|
|
||||||
$output->writeln(sprintf(
|
|
||||||
'<info>Screenshot "%s" is as expected.</info>',
|
|
||||||
$event->getArgument('screenshot')
|
|
||||||
));
|
|
||||||
}
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
$this->eventDispatcher->addListener(
|
|
||||||
'service.screenshot.isDifferent',
|
|
||||||
function (GenericEvent $event) use ($output) {
|
|
||||||
$output->writeln(sprintf(
|
|
||||||
'<error>Screenshot "%s" is different, created diff at "%s".</error>',
|
|
||||||
$event->getArgument('screenshot'),
|
|
||||||
$event->getArgument('diff')
|
|
||||||
));
|
|
||||||
}
|
|
||||||
);
|
|
||||||
|
|
||||||
$this->eventDispatcher->addListener(
|
|
||||||
'service.screenshot.error',
|
|
||||||
function (GenericEvent $event) use ($output) {
|
|
||||||
$output->writeln(sprintf(
|
|
||||||
'<error>"%s"</error>',
|
|
||||||
$event->getArgument('e')->getMessage()
|
|
||||||
));
|
|
||||||
}
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,174 +0,0 @@
|
||||||
<?php
|
|
||||||
|
|
||||||
namespace Codappix\WebsiteComparison\Command;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Copyright (C) 2018 Daniel Siepmann <coding@daniel-siepmann.de>
|
|
||||||
*
|
|
||||||
* This program is free software; you can redistribute it and/or
|
|
||||||
* modify it under the terms of the GNU General Public License
|
|
||||||
* as published by the Free Software Foundation; either version 2
|
|
||||||
* of the License, or (at your option) any later version.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
* GNU General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU General Public License
|
|
||||||
* along with this program; if not, write to the Free Software
|
|
||||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
|
||||||
* 02110-1301, USA.
|
|
||||||
*/
|
|
||||||
|
|
||||||
use Codappix\WebsiteComparison\Model\UrlListDto;
|
|
||||||
use Codappix\WebsiteComparison\Model\UrlListDtoFactory;
|
|
||||||
use Codappix\WebsiteComparison\Service\Screenshot\CrawlerService;
|
|
||||||
use Codappix\WebsiteComparison\Service\Screenshot\Service;
|
|
||||||
use Facebook\WebDriver\Remote\RemoteWebDriver;
|
|
||||||
use Symfony\Component\Console\Command\Command;
|
|
||||||
use Symfony\Component\Console\Input\InputArgument;
|
|
||||||
use Symfony\Component\Console\Input\InputInterface;
|
|
||||||
use Symfony\Component\Console\Input\InputOption;
|
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
|
||||||
use Symfony\Component\EventDispatcher\EventDispatcherInterface;
|
|
||||||
use Symfony\Component\EventDispatcher\GenericEvent;
|
|
||||||
|
|
||||||
class CreateBaseCommand extends Command
|
|
||||||
{
|
|
||||||
/**
|
|
||||||
* @var EventDispatcherInterface
|
|
||||||
*/
|
|
||||||
protected $eventDispatcher;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @var RemoteWebDriver
|
|
||||||
*/
|
|
||||||
protected $webDriver;
|
|
||||||
|
|
||||||
public function __construct(
|
|
||||||
EventDispatcherInterface $eventDispatcher,
|
|
||||||
RemoteWebDriver $webDriver
|
|
||||||
) {
|
|
||||||
parent::__construct(null);
|
|
||||||
|
|
||||||
$this->eventDispatcher = $eventDispatcher;
|
|
||||||
$this->webDriver = $webDriver;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function configure()
|
|
||||||
{
|
|
||||||
$this
|
|
||||||
->setName('comparison:createbase')
|
|
||||||
->setDescription('Creates the base for comparison.')
|
|
||||||
->setHelp('Crawls and screenshots the original website, as a base for future comparison.')
|
|
||||||
|
|
||||||
->addOption(
|
|
||||||
'screenshotDir',
|
|
||||||
null,
|
|
||||||
InputOption::VALUE_OPTIONAL,
|
|
||||||
'Define the sub directory to use for storing created Screenshots.',
|
|
||||||
'output/base'
|
|
||||||
)
|
|
||||||
->addOption(
|
|
||||||
'screenshotWidth',
|
|
||||||
null,
|
|
||||||
InputOption::VALUE_OPTIONAL,
|
|
||||||
'The width for screen resolution and screenshots.',
|
|
||||||
3840
|
|
||||||
)
|
|
||||||
->addOption(
|
|
||||||
'recoverFile',
|
|
||||||
null,
|
|
||||||
InputOption::VALUE_OPTIONAL,
|
|
||||||
'Path to json-File with state of stopped process, used to recover process.',
|
|
||||||
''
|
|
||||||
)
|
|
||||||
|
|
||||||
->addArgument(
|
|
||||||
'baseUrl',
|
|
||||||
InputArgument::REQUIRED,
|
|
||||||
'E.g. https://typo3.org/ the base url of the website to crawl.'
|
|
||||||
)
|
|
||||||
;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function execute(InputInterface $input, OutputInterface $output)
|
|
||||||
{
|
|
||||||
$baseUrl = $input->getArgument('baseUrl');
|
|
||||||
$screenshotDir = $input->getOption('screenshotDir');
|
|
||||||
|
|
||||||
$screenshotService = new Service(
|
|
||||||
$this->eventDispatcher,
|
|
||||||
$screenshotDir,
|
|
||||||
$input->getOption('screenshotWidth')
|
|
||||||
);
|
|
||||||
|
|
||||||
$screenshotCrawler = new CrawlerService(
|
|
||||||
$this->webDriver,
|
|
||||||
$screenshotService,
|
|
||||||
$baseUrl
|
|
||||||
);
|
|
||||||
|
|
||||||
$linkList = $this->getLinkList($baseUrl, $input->getOption('recoverFile'));
|
|
||||||
|
|
||||||
$this->registerEvents($output);
|
|
||||||
try {
|
|
||||||
$screenshotCrawler->crawl($linkList);
|
|
||||||
} catch (\Exception $e) {
|
|
||||||
file_put_contents($this->getJsonFilePath($screenshotService, $baseUrl), json_encode($linkList));
|
|
||||||
$output->writeln(sprintf(
|
|
||||||
'<comment>Saved current state for recovering in "%s".</comment>',
|
|
||||||
$this->getJsonFilePath($screenshotService, $baseUrl)
|
|
||||||
));
|
|
||||||
throw $e;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function getLinkList(
|
|
||||||
string $baseUrl,
|
|
||||||
string $recoverFile = ''
|
|
||||||
): UrlListDto {
|
|
||||||
$factory = new UrlListDtoFactory();
|
|
||||||
|
|
||||||
if (trim($recoverFile) !== '') {
|
|
||||||
return $factory->createWithByConfigurationFile($recoverFile);
|
|
||||||
}
|
|
||||||
|
|
||||||
return $factory->createWithBaseUrl($baseUrl);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function getJsonFilePath(Service $screenshotService, string $baseUrl): string
|
|
||||||
{
|
|
||||||
return $screenshotService->getScreenshotDir() .
|
|
||||||
DIRECTORY_SEPARATOR .
|
|
||||||
$screenshotService->getScreenshotTarget($baseUrl, 'json')
|
|
||||||
;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function registerEvents(OutputInterface $output)
|
|
||||||
{
|
|
||||||
if ($output->isVerbose()) {
|
|
||||||
$this->eventDispatcher->addListener(
|
|
||||||
'service.screenshot.created',
|
|
||||||
function (GenericEvent $event) use ($output) {
|
|
||||||
$output->writeln(sprintf(
|
|
||||||
'<info>Created screenshot "%s" for url "%s".</info>',
|
|
||||||
$event->getArgument('screenshot'),
|
|
||||||
$event->getArgument('url')
|
|
||||||
));
|
|
||||||
}
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
$this->eventDispatcher->addListener(
|
|
||||||
'screenshot.service.error',
|
|
||||||
function (GenericEvent $event) use ($output) {
|
|
||||||
$output->writeln(sprintf(
|
|
||||||
'<error>"%s"</error>',
|
|
||||||
$event->getArgument('e')->getMessage()
|
|
||||||
));
|
|
||||||
}
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,79 +0,0 @@
|
||||||
<?php
|
|
||||||
|
|
||||||
namespace Codappix\WebsiteComparison\Model;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Copyright (C) 2018 Daniel Siepmann <coding@daniel-siepmann.de>
|
|
||||||
*
|
|
||||||
* This program is free software; you can redistribute it and/or
|
|
||||||
* modify it under the terms of the GNU General Public License
|
|
||||||
* as published by the Free Software Foundation; either version 2
|
|
||||||
* of the License, or (at your option) any later version.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
* GNU General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU General Public License
|
|
||||||
* along with this program; if not, write to the Free Software
|
|
||||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
|
||||||
* 02110-1301, USA.
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* List of urls with two states.
|
|
||||||
*
|
|
||||||
* Allows to have a single queue of urls to work on.
|
|
||||||
*/
|
|
||||||
class UrlListDto implements \JsonSerializable
|
|
||||||
{
|
|
||||||
protected $finishedUrls = [];
|
|
||||||
|
|
||||||
protected $upcomingUrls = [];
|
|
||||||
|
|
||||||
public function addUrl(string $url)
|
|
||||||
{
|
|
||||||
if ($this->isUrlKnown($url)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
$this->upcomingUrls[] = $url;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function addFinishedUrl(string $url)
|
|
||||||
{
|
|
||||||
if ($this->isUrlKnown($url)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
$this->finishedUrls[] = $url;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function getNextUrl(): string
|
|
||||||
{
|
|
||||||
return reset($this->upcomingUrls) ?? '';
|
|
||||||
}
|
|
||||||
|
|
||||||
public function markUrlAsFinished(string $url)
|
|
||||||
{
|
|
||||||
$upcomingEntry = array_search($url, $this->upcomingUrls);
|
|
||||||
|
|
||||||
unset($this->upcomingUrls[$upcomingEntry]);
|
|
||||||
|
|
||||||
$this->finishedUrls[] = $url;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function isUrlKnown(string $url): bool
|
|
||||||
{
|
|
||||||
return in_array($url, $this->finishedUrls) || in_array($url, $this->upcomingUrls);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function jsonSerialize()
|
|
||||||
{
|
|
||||||
return [
|
|
||||||
'finishedUrls' => $this->finishedUrls,
|
|
||||||
'upcomingUrls' => $this->upcomingUrls,
|
|
||||||
];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,48 +0,0 @@
|
||||||
<?php
|
|
||||||
|
|
||||||
namespace Codappix\WebsiteComparison\Model;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Copyright (C) 2018 Daniel Siepmann <coding@daniel-siepmann.de>
|
|
||||||
*
|
|
||||||
* This program is free software; you can redistribute it and/or
|
|
||||||
* modify it under the terms of the GNU General Public License
|
|
||||||
* as published by the Free Software Foundation; either version 2
|
|
||||||
* of the License, or (at your option) any later version.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
* GNU General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU General Public License
|
|
||||||
* along with this program; if not, write to the Free Software
|
|
||||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
|
||||||
* 02110-1301, USA.
|
|
||||||
*/
|
|
||||||
|
|
||||||
class UrlListDtoFactory
|
|
||||||
{
|
|
||||||
public function createWithBaseUrl(string $baseUrl): UrlListDto
|
|
||||||
{
|
|
||||||
$urlList = new UrlListDto();
|
|
||||||
$urlList->addUrl($baseUrl);
|
|
||||||
|
|
||||||
return $urlList;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function createWithByConfigurationFile(string $configFile): UrlListDto
|
|
||||||
{
|
|
||||||
$urlList = new UrlListDto();
|
|
||||||
$config = json_decode(file_get_contents($configFile), true);
|
|
||||||
|
|
||||||
foreach ($config['upcomingUrls'] as $url) {
|
|
||||||
$urlList->addUrl($url);
|
|
||||||
}
|
|
||||||
foreach ($config['finishedUrls'] as $url) {
|
|
||||||
$urlList->addFinishedUrl($url);
|
|
||||||
}
|
|
||||||
|
|
||||||
return $urlList;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,152 +0,0 @@
|
||||||
<?php
|
|
||||||
|
|
||||||
namespace Codappix\WebsiteComparison\Service\Screenshot;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Copyright (C) 2018 Daniel Siepmann <coding@daniel-siepmann.de>
|
|
||||||
*
|
|
||||||
* This program is free software; you can redistribute it and/or
|
|
||||||
* modify it under the terms of the GNU General Public License
|
|
||||||
* as published by the Free Software Foundation; either version 2
|
|
||||||
* of the License, or (at your option) any later version.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
* GNU General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU General Public License
|
|
||||||
* along with this program; if not, write to the Free Software
|
|
||||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
|
||||||
* 02110-1301, USA.
|
|
||||||
*/
|
|
||||||
|
|
||||||
use Symfony\Component\EventDispatcher\EventDispatcherInterface;
|
|
||||||
use Symfony\Component\EventDispatcher\GenericEvent;
|
|
||||||
|
|
||||||
class CompareService
|
|
||||||
{
|
|
||||||
/**
|
|
||||||
* @var EventDispatcherInterface
|
|
||||||
*/
|
|
||||||
protected $eventDispatcher;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @var Service
|
|
||||||
*/
|
|
||||||
protected $screenshotService;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @var string
|
|
||||||
*/
|
|
||||||
protected $compareDirectory = '';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @var string
|
|
||||||
*/
|
|
||||||
protected $diffResultDir = '';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @var bool
|
|
||||||
*/
|
|
||||||
protected $hasDifferences = false;
|
|
||||||
|
|
||||||
public function __construct(
|
|
||||||
EventDispatcherInterface $eventDispatcher,
|
|
||||||
Service $screenshotService,
|
|
||||||
string $compareDirectory,
|
|
||||||
string $diffResultDir
|
|
||||||
) {
|
|
||||||
$this->eventDispatcher = $eventDispatcher;
|
|
||||||
$this->screenshotService = $screenshotService;
|
|
||||||
$this->compareDirectory = $this->screenshotService->convertReltiveFolder($compareDirectory);
|
|
||||||
$this->diffResultDir = $this->screenshotService->convertReltiveFolder($diffResultDir);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function hasDifferences(): bool
|
|
||||||
{
|
|
||||||
return $this->hasDifferences;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function compareScreenshot(string $screenshot)
|
|
||||||
{
|
|
||||||
try {
|
|
||||||
if ($this->doScreenshotsDiffer($screenshot)) {
|
|
||||||
$this->eventDispatcher->dispatch(
|
|
||||||
'service.screenshot.isDifferent',
|
|
||||||
new GenericEvent('New Screenshot is different then base version.', [
|
|
||||||
'screenshot' => $screenshot,
|
|
||||||
'diff' => $this->getDiffFileName($screenshot)
|
|
||||||
])
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
} catch (\ImagickException $e) {
|
|
||||||
$this->eventDispatcher->dispatch(
|
|
||||||
'service.screenshot.error',
|
|
||||||
new GenericEvent($e->getMessage(), [$e])
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
$this->eventDispatcher->dispatch(
|
|
||||||
'service.screenshot.isSame',
|
|
||||||
new GenericEvent('New Screenshot is the same as base version.', [
|
|
||||||
'screenshot' => $screenshot,
|
|
||||||
])
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function doScreenshotsDiffer(string $screenshot): bool
|
|
||||||
{
|
|
||||||
$actualScreenshot = new \Imagick($screenshot);
|
|
||||||
$actualGeometry = $actualScreenshot->getImageGeometry();
|
|
||||||
$compareScreenshot = new \Imagick($this->getBaseScreenshot($screenshot));
|
|
||||||
$compareGeometry = $compareScreenshot->getImageGeometry();
|
|
||||||
|
|
||||||
if ($actualGeometry !== $compareGeometry) {
|
|
||||||
throw new \ImagickException(sprintf(
|
|
||||||
"Screenshots don't have an equal geometry. Should be %sx%s but is %sx%s",
|
|
||||||
$compareGeometry['width'],
|
|
||||||
$compareGeometry['height'],
|
|
||||||
$actualGeometry['width'],
|
|
||||||
$actualGeometry['height']
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
$result = $actualScreenshot->compareImages(
|
|
||||||
$compareScreenshot,
|
|
||||||
\Imagick::METRIC_ROOTMEANSQUAREDERROR
|
|
||||||
);
|
|
||||||
if ($result[1] > 0) {
|
|
||||||
/** @var \Imagick $diffScreenshot */
|
|
||||||
$diffScreenshot = $result[0];
|
|
||||||
$diffScreenshot->setImageFormat('png');
|
|
||||||
$fileName = $this->getDiffFileName($screenshot);
|
|
||||||
$this->screenshotService->createDir(dirname($fileName));
|
|
||||||
file_put_contents($fileName, $diffScreenshot);
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function getBaseScreenshot(string $compareScreenshot): string
|
|
||||||
{
|
|
||||||
return str_replace(
|
|
||||||
$this->screenshotService->getScreenshotDir(),
|
|
||||||
$this->compareDirectory,
|
|
||||||
$compareScreenshot
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function getDiffFileName(string $screenshot): string
|
|
||||||
{
|
|
||||||
return str_replace(
|
|
||||||
$this->screenshotService->getScreenshotDir(),
|
|
||||||
$this->diffResultDir,
|
|
||||||
$screenshot
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,138 +0,0 @@
|
||||||
<?php
|
|
||||||
|
|
||||||
namespace Codappix\WebsiteComparison\Service\Screenshot;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Copyright (C) 2018 Daniel Siepmann <coding@daniel-siepmann.de>
|
|
||||||
*
|
|
||||||
* This program is free software; you can redistribute it and/or
|
|
||||||
* modify it under the terms of the GNU General Public License
|
|
||||||
* as published by the Free Software Foundation; either version 2
|
|
||||||
* of the License, or (at your option) any later version.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
* GNU General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU General Public License
|
|
||||||
* along with this program; if not, write to the Free Software
|
|
||||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
|
||||||
* 02110-1301, USA.
|
|
||||||
*/
|
|
||||||
|
|
||||||
use Codappix\WebsiteComparison\Model\UrlListDto;
|
|
||||||
use Facebook\WebDriver\Remote\RemoteWebDriver;
|
|
||||||
use Facebook\WebDriver\Remote\RemoteWebElement;
|
|
||||||
use Facebook\WebDriver\WebDriverBy;
|
|
||||||
use GuzzleHttp\Psr7\Uri;
|
|
||||||
|
|
||||||
class CrawlerService
|
|
||||||
{
|
|
||||||
/**
|
|
||||||
* @var RemoteWebDriver
|
|
||||||
*/
|
|
||||||
protected $driver;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @var Service
|
|
||||||
*/
|
|
||||||
protected $screenshotService;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @var string
|
|
||||||
*/
|
|
||||||
protected $baseUrl = '';
|
|
||||||
|
|
||||||
public function __construct(
|
|
||||||
RemoteWebDriver $driver,
|
|
||||||
Service $screenshotService,
|
|
||||||
string $baseUrl
|
|
||||||
) {
|
|
||||||
$this->driver = $driver;
|
|
||||||
$this->screenshotService = $screenshotService;
|
|
||||||
$this->baseUrl = rtrim($baseUrl, '/') . '/';
|
|
||||||
}
|
|
||||||
|
|
||||||
public function crawl(UrlListDto $linkList)
|
|
||||||
{
|
|
||||||
while ($url = $linkList->getNextUrl()) {
|
|
||||||
$uri = new Uri($url);
|
|
||||||
$this->driver->get($url);
|
|
||||||
$screenshotHeight = $this->driver->findElement(WebDriverBy::cssSelector('body'))
|
|
||||||
->getSize()
|
|
||||||
->getHeight();
|
|
||||||
$this->screenshotService->createScreenshot(
|
|
||||||
$this->driver->getCurrentURL(),
|
|
||||||
$screenshotHeight
|
|
||||||
);
|
|
||||||
|
|
||||||
$linkList->markUrlAsFinished($url);
|
|
||||||
array_map([$linkList, 'addUrl'], $this->fetchFurtherLinks(
|
|
||||||
$this->driver->findElements(WebDriverBy::cssSelector('a'))
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function fetchFurtherLinks(array $webElements): array
|
|
||||||
{
|
|
||||||
$links = [];
|
|
||||||
foreach ($webElements as $webElement) {
|
|
||||||
try {
|
|
||||||
$link = $this->fetchLinkFromElement($webElement);
|
|
||||||
} catch (\Exception $e) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
$links[] = $link;
|
|
||||||
}
|
|
||||||
|
|
||||||
return $links;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function fetchLinkFromElement(RemoteWebElement $element): string
|
|
||||||
{
|
|
||||||
$uri = null;
|
|
||||||
$href = $element->getAttribute('href');
|
|
||||||
if (is_string($href)) {
|
|
||||||
$uri = new Uri($href);
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($uri === null) {
|
|
||||||
throw new \Exception('Did not get a Uri for element.', 1535530859);
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($this->isInternalLink($uri)) {
|
|
||||||
return (string) $uri;
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new \Exception('Was external link.', 1535639056);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function isInternalLink(Uri $uri): bool
|
|
||||||
{
|
|
||||||
$validHosts = [
|
|
||||||
'',
|
|
||||||
(new Uri($this->baseUrl))->getHost(),
|
|
||||||
];
|
|
||||||
$validSchemas = [
|
|
||||||
'http',
|
|
||||||
'https',
|
|
||||||
];
|
|
||||||
|
|
||||||
$invalidFileExtensions = [
|
|
||||||
'.pdf',
|
|
||||||
'.jpg',
|
|
||||||
'.gif',
|
|
||||||
'.svg',
|
|
||||||
];
|
|
||||||
|
|
||||||
$pathEnding = substr($uri->getPath(), -4);
|
|
||||||
|
|
||||||
return in_array($uri->getHost(), $validHosts)
|
|
||||||
&& in_array($uri->getScheme(), $validSchemas)
|
|
||||||
&& !in_array($pathEnding, $invalidFileExtensions)
|
|
||||||
&& strpos((string) $uri, 'eID=') === false
|
|
||||||
;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,133 +0,0 @@
|
||||||
<?php
|
|
||||||
|
|
||||||
namespace Codappix\WebsiteComparison\Service\Screenshot;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Copyright (C) 2018 Daniel Siepmann <coding@daniel-siepmann.de>
|
|
||||||
*
|
|
||||||
* This program is free software; you can redistribute it and/or
|
|
||||||
* modify it under the terms of the GNU General Public License
|
|
||||||
* as published by the Free Software Foundation; either version 2
|
|
||||||
* of the License, or (at your option) any later version.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
* GNU General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU General Public License
|
|
||||||
* along with this program; if not, write to the Free Software
|
|
||||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
|
||||||
* 02110-1301, USA.
|
|
||||||
*/
|
|
||||||
|
|
||||||
use GuzzleHttp\Psr7\Uri;
|
|
||||||
use Symfony\Component\EventDispatcher\EventDispatcherInterface;
|
|
||||||
use Symfony\Component\EventDispatcher\GenericEvent;
|
|
||||||
use Symfony\Component\Process\Process;
|
|
||||||
|
|
||||||
class Service
|
|
||||||
{
|
|
||||||
/**
|
|
||||||
* @var EventDispatcherInterface
|
|
||||||
*/
|
|
||||||
protected $eventDispatcher;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @var string
|
|
||||||
*/
|
|
||||||
protected $screenshotDir = '';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @var int
|
|
||||||
*/
|
|
||||||
protected $screenshotWidth = 3840;
|
|
||||||
|
|
||||||
public function __construct(
|
|
||||||
EventDispatcherInterface $eventDispatcher,
|
|
||||||
string $screenshotDir,
|
|
||||||
int $screenshotWidth
|
|
||||||
) {
|
|
||||||
$this->eventDispatcher = $eventDispatcher;
|
|
||||||
$this->screenshotDir = $this->convertReltiveFolder($screenshotDir);
|
|
||||||
$this->screenshotWidth = $screenshotWidth;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function createScreenshot(string $url, int $height): string
|
|
||||||
{
|
|
||||||
// TODO: Include width in screenshot dir.
|
|
||||||
// This enables to compare different resolutions
|
|
||||||
$screenshotTarget = $this->getScreenshotTarget($url);
|
|
||||||
$this->createDir($this->screenshotDir . dirname($screenshotTarget));
|
|
||||||
$completeScreenshotTarget = $this->screenshotDir . $screenshotTarget;
|
|
||||||
|
|
||||||
$screenshotProcess = new Process([
|
|
||||||
'chromium-browser',
|
|
||||||
'--headless',
|
|
||||||
'--disable-gpu',
|
|
||||||
'--window-size=' . $this->screenshotWidth . ',' . $height,
|
|
||||||
'--screenshot=' . $completeScreenshotTarget,
|
|
||||||
$url
|
|
||||||
]);
|
|
||||||
$screenshotProcess->setTimeout(60 * 2);
|
|
||||||
// TODO: Check for success
|
|
||||||
$screenshotProcess->run();
|
|
||||||
|
|
||||||
$this->eventDispatcher->dispatch(
|
|
||||||
'service.screenshot.created',
|
|
||||||
new GenericEvent('Created Screenshot', [
|
|
||||||
'screenshot' => $completeScreenshotTarget,
|
|
||||||
'url' => $url,
|
|
||||||
])
|
|
||||||
);
|
|
||||||
|
|
||||||
return $completeScreenshotTarget;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function getScreenshotTarget(string $url, string $suffix = 'png'): string
|
|
||||||
{
|
|
||||||
$uri = new Uri($url);
|
|
||||||
|
|
||||||
return implode(
|
|
||||||
DIRECTORY_SEPARATOR,
|
|
||||||
array_filter(
|
|
||||||
[
|
|
||||||
$uri->getScheme(),
|
|
||||||
$uri->getHost(),
|
|
||||||
trim($uri->getPath(), '/'),
|
|
||||||
$uri->getQuery(),
|
|
||||||
],
|
|
||||||
function (string $string) {
|
|
||||||
return trim($string, ' /') !== '';
|
|
||||||
}
|
|
||||||
)
|
|
||||||
) . '.' . $suffix;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function getScreenshotDir(): string
|
|
||||||
{
|
|
||||||
return $this->screenshotDir;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @throws \Exception If folder could not be created.
|
|
||||||
*/
|
|
||||||
public function createDir(string $dir)
|
|
||||||
{
|
|
||||||
if (!is_dir($dir)) {
|
|
||||||
mkdir($dir, 0777, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!is_dir($dir)) {
|
|
||||||
throw new \Exception('Could not create directory: "' . $dir . '".', 1535528875);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public function convertReltiveFolder(string $folder): string
|
|
||||||
{
|
|
||||||
return implode(DIRECTORY_SEPARATOR, [
|
|
||||||
dirname(dirname(dirname(dirname(__FILE__)))),
|
|
||||||
rtrim($folder, '/'),
|
|
||||||
]) . DIRECTORY_SEPARATOR;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Loading…
Reference in a new issue