TASK: Filter unnecessary urls.

Do not include output logic into services.
Also do not couple comparison with crawling.

Use Symfony Events to connect features.

.gitignore

@@ -0,0 +1,3 @@
+/vendor/
+/composer.lock
+/output/

comparison

@@ -0,0 +1,42 @@
+#!/usr/bin/env php
+<?php
+
+declare(ticks = 1);
+
+pcntl_signal(SIGINT, function () {
+    exit(99);
+});
+
+require __DIR__ . '/vendor/autoload.php';
+
+use Codappix\WebsiteComparison\Command\CompareCommand;
+use Codappix\WebsiteComparison\Command\CreateBaseCommand;
+use Facebook\WebDriver\Chrome\ChromeDriver;
+use Facebook\WebDriver\Chrome\ChromeDriverService;
+use Symfony\Component\Console\Application;
+use Symfony\Component\EventDispatcher\EventDispatcher;
+
+$eventDispatcher = new EventDispatcher();
+
+$chromeDriver = (function () {
+    $chromeDriverService = new ChromeDriverService(
+        '/usr/lib/chromium-browser/chromedriver',
+        9515,
+        [
+            '--port=9515',
+            '--headless',
+        ]
+    );
+
+    return ChromeDriver::start(null, $chromeDriverService);
+})();
+
+
+$application = new Application();
+$application->setDispatcher($eventDispatcher);
+
+// TODO: Use factory for commands, which injects event dispatcher and chrome driver?
+$application->add(new CreateBaseCommand($eventDispatcher, $chromeDriver));
+$application->add(new CompareCommand($eventDispatcher, $chromeDriver));
+
+$application->run();

composer.json

@@ -0,0 +1,26 @@
+{
+    "name": "codappix/website-comparison",
+    "description": "Compares a Website visually by comparing Screenshots.",
+    "type": "project",
+    "license": "GPL-2.0-or-later",
+    "authors": [
+        {
+            "name": "Daniel Siepmann",
+            "email": "coding@daniel-siepmann.de"
+        }
+    ],
+    "autoload": {
+        "psr-4": {
+            "Codappix\\WebsiteComparison\\": "src/"
+        }
+    },
+    "require": {
+        "php": "^7.2",
+        "ext-imagick": "*",
+        "facebook/webdriver": "^1.6",
+        "symfony/console": "^4.1",
+        "symfony/process": "^4.1",
+        "guzzlehttp/psr7": "^1.4",
+        "symfony/event-dispatcher": "^4.1"
+    }
+}

readme.rst

@@ -0,0 +1,26 @@
+Website Comparison
+==================
+
+A small command line tool that will screenshot a whole website. On the next run
+there can be a comparison to the base that was created before.
+
+Installation
+------------
+
+Run ``composer install``.
+
+Also ``chromedriver`` is needed. Right now the pat his hardcoded to
+``/usr/lib/chromium-browser/chromedriver`` in file ``comparison``.
+
+Also the ``chromium-browser`` binary has to be inside the ``$PATH``.
+
+Usage
+-----
+
+First run ``./comparison comparison:createbase`` to create the "base" version of the
+website. This way we have the base to compare against later.
+
+Second run ``./comparison comparison:comparetobase`` to screenshot website again and
+inform about differences.
+
+For further possible options add ``--help``.

src/Command/CompareCommand.php

@@ -0,0 +1,222 @@
+<?php
+
+namespace Codappix\WebsiteComparison\Command;
+
+/*
+ * Copyright (C) 2018 Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\WebsiteComparison\Model\UrlListDto;
+use Codappix\WebsiteComparison\Model\UrlListDtoFactory;
+use Codappix\WebsiteComparison\Service\Screenshot\CompareService;
+use Codappix\WebsiteComparison\Service\Screenshot\CrawlerService;
+use Codappix\WebsiteComparison\Service\Screenshot\Service;
+use Facebook\WebDriver\Remote\RemoteWebDriver;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\EventDispatcher\EventDispatcherInterface;
+use Symfony\Component\EventDispatcher\GenericEvent;
+
+class CompareCommand extends Command
+{
+    /**
+     * @var EventDispatcherInterface
+     */
+    protected $eventDispatcher;
+
+    /**
+     * @var RemoteWebDriver
+     */
+    protected $webDriver;
+
+    public function __construct(
+        EventDispatcherInterface $eventDispatcher,
+        RemoteWebDriver $webDriver
+    ) {
+        parent::__construct(null);
+
+        $this->eventDispatcher = $eventDispatcher;
+        $this->webDriver = $webDriver;
+    }
+
+    protected function configure()
+    {
+        $this
+            ->setName('comparison:comparetobase')
+            ->setDescription('Compare curent state against saved base.')
+            ->setHelp('Crawls and screenshots the original website, as a base for future comparison.')
+
+            ->addOption(
+                'screenshotDir',
+                null,
+                InputOption::VALUE_OPTIONAL,
+                'Define the sub directory containing original Screenshots for comparison.',
+                'output/base'
+            )
+            ->addOption(
+                'compareDir',
+                null,
+                InputOption::VALUE_OPTIONAL,
+                'Define the sub directory to use for storing created Screenshots.',
+                'output/compare'
+            )
+            ->addOption(
+                'diffResultDir',
+                null,
+                InputOption::VALUE_OPTIONAL,
+                'Define the sub directory to use for storing created diffs.',
+                'output/diffResult'
+            )
+            ->addOption(
+                'screenshotWidth',
+                null,
+                InputOption::VALUE_OPTIONAL,
+                'The width for screen resolution and screenshots.',
+                3840
+            )
+            ->addOption(
+                'recoverFile',
+                null,
+                InputOption::VALUE_OPTIONAL,
+                'Path to json-File with state of stopped process, used to recover process.',
+                ''
+            )
+
+            ->addArgument(
+                'baseUrl',
+                InputArgument::REQUIRED,
+                'E.g. https://typo3.org/ the base url of the website to crawl.'
+            )
+        ;
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        $baseUrl = $input->getArgument('baseUrl');
+
+        $screenshotService = new Service(
+            $this->eventDispatcher,
+            $input->getOption('compareDir'),
+            $input->getOption('screenshotWidth')
+        );
+
+        $screenshotCrawler = new CrawlerService(
+            $this->webDriver,
+            $screenshotService,
+            $baseUrl
+        );
+
+        $compareService = new CompareService(
+            $this->eventDispatcher,
+            $screenshotService,
+            $input->getOption('screenshotDir'),
+            $input->getOption('diffResultDir')
+        );
+
+        $linkList = $this->getLinkList($baseUrl, $input->getOption('recoverFile'));
+
+        $this->registerEvents($output, $compareService);
+        try {
+            $screenshotCrawler->crawl($linkList);
+        } catch (\Exception $e) {
+            file_put_contents($this->getJsonFilePath($screenshotService, $baseUrl), json_encode($linkList));
+            $output->writeln(sprintf(
+                '<comment>Saved current state for recovering in "%s".</comment>',
+                $this->getJsonFilePath($screenshotService, $baseUrl)
+            ));
+            throw $e;
+        }
+
+        if ($compareService->hasDifferences()) {
+            return 255;
+        }
+    }
+
+    protected function getLinkList(
+        string $baseUrl,
+        string $recoverFile = ''
+    ): UrlListDto {
+        $factory = new UrlListDtoFactory();
+
+        if (trim($recoverFile) !== '') {
+            return $factory->createWithByConfigurationFile($recoverFile);
+        }
+
+        return $factory->createWithBaseUrl($baseUrl);
+    }
+
+    protected function getJsonFilePath(Service $screenshotService, string $baseUrl): string
+    {
+        return $screenshotService->getScreenshotDir() .
+            DIRECTORY_SEPARATOR .
+            $screenshotService->getScreenshotTarget($baseUrl, 'json')
+            ;
+    }
+
+    protected function registerEvents(OutputInterface $output, CompareService $compareService)
+    {
+        $this->eventDispatcher->addListener(
+            'service.screenshot.created',
+            function (GenericEvent $event) use ($output, $compareService) {
+                $output->writeln(sprintf(
+                    '<info>Comparing Screenshot for url "%s".</info>',
+                    $event->getArgument('url')
+                ));
+                $compareService->compareScreenshot(
+                    $event->getArgument('screenshot')
+                );
+            }
+        );
+
+        if ($output->isVerbose()) {
+            $this->eventDispatcher->addListener(
+                'service.screenshot.isSame',
+                function (GenericEvent $event) use ($output) {
+                    $output->writeln(sprintf(
+                        '<info>Screenshot "%s" is as expected.</info>',
+                        $event->getArgument('screenshot')
+                    ));
+                }
+            );
+        }
+
+        $this->eventDispatcher->addListener(
+            'service.screenshot.isDifferent',
+            function (GenericEvent $event) use ($output) {
+                $output->writeln(sprintf(
+                    '<error>Screenshot "%s" is different, created diff at "%s".</error>',
+                    $event->getArgument('screenshot'),
+                    $event->getArgument('diff')
+                ));
+            }
+        );
+
+        $this->eventDispatcher->addListener(
+            'service.screenshot.error',
+            function (GenericEvent $event) use ($output) {
+                $output->writeln(sprintf(
+                    '<error>"%s"</error>',
+                    $event->getArgument('e')->getMessage()
+                ));
+            }
+        );
+    }
+}

src/Command/CreateBaseCommand.php

@@ -0,0 +1,174 @@
+<?php
+
+namespace Codappix\WebsiteComparison\Command;
+
+/*
+ * Copyright (C) 2018 Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\WebsiteComparison\Model\UrlListDto;
+use Codappix\WebsiteComparison\Model\UrlListDtoFactory;
+use Codappix\WebsiteComparison\Service\Screenshot\CrawlerService;
+use Codappix\WebsiteComparison\Service\Screenshot\Service;
+use Facebook\WebDriver\Remote\RemoteWebDriver;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\EventDispatcher\EventDispatcherInterface;
+use Symfony\Component\EventDispatcher\GenericEvent;
+
+class CreateBaseCommand extends Command
+{
+    /**
+     * @var EventDispatcherInterface
+     */
+    protected $eventDispatcher;
+
+    /**
+     * @var RemoteWebDriver
+     */
+    protected $webDriver;
+
+    public function __construct(
+        EventDispatcherInterface $eventDispatcher,
+        RemoteWebDriver $webDriver
+    ) {
+        parent::__construct(null);
+
+        $this->eventDispatcher = $eventDispatcher;
+        $this->webDriver = $webDriver;
+    }
+
+    protected function configure()
+    {
+        $this
+            ->setName('comparison:createbase')
+            ->setDescription('Creates the base for comparison.')
+            ->setHelp('Crawls and screenshots the original website, as a base for future comparison.')
+
+            ->addOption(
+                'screenshotDir',
+                null,
+                InputOption::VALUE_OPTIONAL,
+                'Define the sub directory to use for storing created Screenshots.',
+                'output/base'
+            )
+            ->addOption(
+                'screenshotWidth',
+                null,
+                InputOption::VALUE_OPTIONAL,
+                'The width for screen resolution and screenshots.',
+                3840
+            )
+            ->addOption(
+                'recoverFile',
+                null,
+                InputOption::VALUE_OPTIONAL,
+                'Path to json-File with state of stopped process, used to recover process.',
+                ''
+            )
+
+            ->addArgument(
+                'baseUrl',
+                InputArgument::REQUIRED,
+                'E.g. https://typo3.org/ the base url of the website to crawl.'
+            )
+        ;
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        $baseUrl = $input->getArgument('baseUrl');
+        $screenshotDir = $input->getOption('screenshotDir');
+
+        $screenshotService = new Service(
+            $this->eventDispatcher,
+            $screenshotDir,
+            $input->getOption('screenshotWidth')
+        );
+
+        $screenshotCrawler = new CrawlerService(
+            $this->webDriver,
+            $screenshotService,
+            $baseUrl
+        );
+
+        $linkList = $this->getLinkList($baseUrl, $input->getOption('recoverFile'));
+
+        $this->registerEvents($output);
+        try {
+            $screenshotCrawler->crawl($linkList);
+        } catch (\Exception $e) {
+            file_put_contents($this->getJsonFilePath($screenshotService, $baseUrl), json_encode($linkList));
+            $output->writeln(sprintf(
+                '<comment>Saved current state for recovering in "%s".</comment>',
+                $this->getJsonFilePath($screenshotService, $baseUrl)
+            ));
+            throw $e;
+        }
+    }
+
+    protected function getLinkList(
+        string $baseUrl,
+        string $recoverFile = ''
+    ): UrlListDto {
+        $factory = new UrlListDtoFactory();
+
+        if (trim($recoverFile) !== '') {
+            return $factory->createWithByConfigurationFile($recoverFile);
+        }
+
+        return $factory->createWithBaseUrl($baseUrl);
+    }
+
+    protected function getJsonFilePath(Service $screenshotService, string $baseUrl): string
+    {
+        return $screenshotService->getScreenshotDir() .
+            DIRECTORY_SEPARATOR .
+            $screenshotService->getScreenshotTarget($baseUrl, 'json')
+            ;
+    }
+
+    protected function registerEvents(OutputInterface $output)
+    {
+        if ($output->isVerbose()) {
+            $this->eventDispatcher->addListener(
+                'service.screenshot.created',
+                function (GenericEvent $event) use ($output) {
+                    $output->writeln(sprintf(
+                        '<info>Created screenshot "%s" for url "%s".</info>',
+                        $event->getArgument('screenshot'),
+                        $event->getArgument('url')
+                    ));
+                }
+            );
+        }
+
+        $this->eventDispatcher->addListener(
+            'screenshot.service.error',
+            function (GenericEvent $event) use ($output) {
+                $output->writeln(sprintf(
+                    '<error>"%s"</error>',
+                    $event->getArgument('e')->getMessage()
+                ));
+            }
+        );
+    }
+}

src/Model/UrlListDto.php

@@ -0,0 +1,79 @@
+<?php
+
+namespace Codappix\WebsiteComparison\Model;
+
+/*
+ * Copyright (C) 2018 Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * List of urls with two states.
+ *
+ * Allows to have a single queue of urls to work on.
+ */
+class UrlListDto implements \JsonSerializable
+{
+    protected $finishedUrls = [];
+
+    protected $upcomingUrls = [];
+
+    public function addUrl(string $url)
+    {
+        if ($this->isUrlKnown($url)) {
+            return;
+        }
+
+        $this->upcomingUrls[] = $url;
+    }
+
+    public function addFinishedUrl(string $url)
+    {
+        if ($this->isUrlKnown($url)) {
+            return;
+        }
+
+        $this->finishedUrls[] = $url;
+    }
+
+    public function getNextUrl(): string
+    {
+        return reset($this->upcomingUrls) ?? '';
+    }
+
+    public function markUrlAsFinished(string $url)
+    {
+        $upcomingEntry = array_search($url, $this->upcomingUrls);
+
+        unset($this->upcomingUrls[$upcomingEntry]);
+
+        $this->finishedUrls[] = $url;
+    }
+
+    public function isUrlKnown(string $url): bool
+    {
+        return in_array($url, $this->finishedUrls) || in_array($url, $this->upcomingUrls);
+    }
+
+    public function jsonSerialize()
+    {
+        return [
+            'finishedUrls' => $this->finishedUrls,
+            'upcomingUrls' => $this->upcomingUrls,
+        ];
+    }
+}

src/Model/UrlListDtoFactory.php

@@ -0,0 +1,48 @@
+<?php
+
+namespace Codappix\WebsiteComparison\Model;
+
+/*
+ * Copyright (C) 2018 Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+class UrlListDtoFactory
+{
+    public function createWithBaseUrl(string $baseUrl): UrlListDto
+    {
+        $urlList = new UrlListDto();
+        $urlList->addUrl($baseUrl);
+
+        return $urlList;
+    }
+
+    public function createWithByConfigurationFile(string $configFile): UrlListDto
+    {
+        $urlList = new UrlListDto();
+        $config = json_decode(file_get_contents($configFile), true);
+
+        foreach ($config['upcomingUrls'] as $url) {
+            $urlList->addUrl($url);
+        }
+        foreach ($config['finishedUrls'] as $url) {
+            $urlList->addFinishedUrl($url);
+        }
+
+        return $urlList;
+    }
+}

src/Service/Screenshot/CompareService.php

@@ -0,0 +1,152 @@
+<?php
+
+namespace Codappix\WebsiteComparison\Service\Screenshot;
+
+/*
+ * Copyright (C) 2018 Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Symfony\Component\EventDispatcher\EventDispatcherInterface;
+use Symfony\Component\EventDispatcher\GenericEvent;
+
+class CompareService
+{
+    /**
+     * @var EventDispatcherInterface
+     */
+    protected $eventDispatcher;
+
+    /**
+     * @var Service
+     */
+    protected $screenshotService;
+
+    /**
+     * @var string
+     */
+    protected $compareDirectory = '';
+
+    /**
+     * @var string
+     */
+    protected $diffResultDir = '';
+
+    /**
+     * @var bool
+     */
+    protected $hasDifferences = false;
+
+    public function __construct(
+        EventDispatcherInterface $eventDispatcher,
+        Service $screenshotService,
+        string $compareDirectory,
+        string $diffResultDir
+    ) {
+        $this->eventDispatcher = $eventDispatcher;
+        $this->screenshotService = $screenshotService;
+        $this->compareDirectory = $this->screenshotService->convertReltiveFolder($compareDirectory);
+        $this->diffResultDir = $this->screenshotService->convertReltiveFolder($diffResultDir);
+    }
+
+    public function hasDifferences(): bool
+    {
+        return $this->hasDifferences;
+    }
+
+    public function compareScreenshot(string $screenshot)
+    {
+        try {
+            if ($this->doScreenshotsDiffer($screenshot)) {
+                $this->eventDispatcher->dispatch(
+                    'service.screenshot.isDifferent',
+                    new GenericEvent('New Screenshot is different then base version.', [
+                        'screenshot' => $screenshot,
+                        'diff' => $this->getDiffFileName($screenshot)
+                    ])
+                );
+                return;
+            }
+        } catch (\ImagickException $e) {
+            $this->eventDispatcher->dispatch(
+                'service.screenshot.error',
+                new GenericEvent($e->getMessage(), [$e])
+            );
+            return;
+        }
+
+        $this->eventDispatcher->dispatch(
+            'service.screenshot.isSame',
+            new GenericEvent('New Screenshot is the same as base version.', [
+                'screenshot' => $screenshot,
+            ])
+        );
+    }
+
+    protected function doScreenshotsDiffer(string $screenshot): bool
+    {
+        $actualScreenshot = new \Imagick($screenshot);
+        $actualGeometry = $actualScreenshot->getImageGeometry();
+        $compareScreenshot = new \Imagick($this->getBaseScreenshot($screenshot));
+        $compareGeometry = $compareScreenshot->getImageGeometry();
+
+        if ($actualGeometry !== $compareGeometry) {
+            throw new \ImagickException(sprintf(
+                "Screenshots don't have an equal geometry. Should be %sx%s but is %sx%s",
+                $compareGeometry['width'],
+                $compareGeometry['height'],
+                $actualGeometry['width'],
+                $actualGeometry['height']
+            ));
+        }
+
+        $result = $actualScreenshot->compareImages(
+            $compareScreenshot,
+            \Imagick::METRIC_ROOTMEANSQUAREDERROR
+        );
+        if ($result[1] > 0) {
+            /** @var \Imagick $diffScreenshot */
+            $diffScreenshot = $result[0];
+            $diffScreenshot->setImageFormat('png');
+            $fileName = $this->getDiffFileName($screenshot);
+            $this->screenshotService->createDir(dirname($fileName));
+            file_put_contents($fileName, $diffScreenshot);
+
+            return true;
+        }
+
+        return false;
+    }
+
+    protected function getBaseScreenshot(string $compareScreenshot): string
+    {
+        return str_replace(
+            $this->screenshotService->getScreenshotDir(),
+            $this->compareDirectory,
+            $compareScreenshot
+        );
+    }
+
+    protected function getDiffFileName(string $screenshot): string
+    {
+        return str_replace(
+            $this->screenshotService->getScreenshotDir(),
+            $this->diffResultDir,
+            $screenshot
+        );
+    }
+}

src/Service/Screenshot/CrawlerService.php

@@ -0,0 +1,138 @@
+<?php
+
+namespace Codappix\WebsiteComparison\Service\Screenshot;
+
+/*
+ * Copyright (C) 2018 Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use Codappix\WebsiteComparison\Model\UrlListDto;
+use Facebook\WebDriver\Remote\RemoteWebDriver;
+use Facebook\WebDriver\Remote\RemoteWebElement;
+use Facebook\WebDriver\WebDriverBy;
+use GuzzleHttp\Psr7\Uri;
+
+class CrawlerService
+{
+    /**
+     * @var RemoteWebDriver
+     */
+    protected $driver;
+
+    /**
+     * @var Service
+     */
+    protected $screenshotService;
+
+    /**
+     * @var string
+     */
+    protected $baseUrl = '';
+
+    public function __construct(
+        RemoteWebDriver $driver,
+        Service $screenshotService,
+        string $baseUrl
+    ) {
+        $this->driver = $driver;
+        $this->screenshotService = $screenshotService;
+        $this->baseUrl = rtrim($baseUrl, '/') . '/';
+    }
+
+    public function crawl(UrlListDto $linkList)
+    {
+        while ($url = $linkList->getNextUrl()) {
+            $uri = new Uri($url);
+            $this->driver->get($url);
+            $screenshotHeight = $this->driver->findElement(WebDriverBy::cssSelector('body'))
+                ->getSize()
+                ->getHeight();
+            $this->screenshotService->createScreenshot(
+                $this->driver->getCurrentURL(),
+                $screenshotHeight
+            );
+
+            $linkList->markUrlAsFinished($url);
+            array_map([$linkList, 'addUrl'], $this->fetchFurtherLinks(
+                $this->driver->findElements(WebDriverBy::cssSelector('a'))
+            ));
+        }
+    }
+
+    protected function fetchFurtherLinks(array $webElements): array
+    {
+        $links = [];
+        foreach ($webElements as $webElement) {
+            try {
+                $link = $this->fetchLinkFromElement($webElement);
+            } catch (\Exception $e) {
+                continue;
+            }
+
+            $links[] = $link;
+        }
+
+        return $links;
+    }
+
+    protected function fetchLinkFromElement(RemoteWebElement $element): string
+    {
+        $uri = null;
+        $href = $element->getAttribute('href');
+        if (is_string($href)) {
+            $uri = new Uri($href);
+        }
+
+        if ($uri === null) {
+            throw new \Exception('Did not get a Uri for element.', 1535530859);
+        }
+
+        if ($this->isInternalLink($uri)) {
+            return (string) $uri;
+        }
+
+        throw new \Exception('Was external link.', 1535639056);
+    }
+
+    protected function isInternalLink(Uri $uri): bool
+    {
+        $validHosts = [
+            '',
+            (new Uri($this->baseUrl))->getHost(),
+        ];
+        $validSchemas = [
+            'http',
+            'https',
+        ];
+
+        $invalidFileExtensions = [
+            '.pdf',
+            '.jpg',
+            '.gif',
+            '.svg',
+        ];
+
+        $pathEnding = substr($uri->getPath(), -4);
+
+        return in_array($uri->getHost(), $validHosts)
+            && in_array($uri->getScheme(), $validSchemas)
+            && !in_array($pathEnding, $invalidFileExtensions)
+            && strpos((string) $uri, 'eID=') === false
+            ;
+    }
+}

src/Service/Screenshot/Service.php

@@ -0,0 +1,133 @@
+<?php
+
+namespace Codappix\WebsiteComparison\Service\Screenshot;
+
+/*
+ * Copyright (C) 2018 Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+use GuzzleHttp\Psr7\Uri;
+use Symfony\Component\EventDispatcher\EventDispatcherInterface;
+use Symfony\Component\EventDispatcher\GenericEvent;
+use Symfony\Component\Process\Process;
+
+class Service
+{
+    /**
+     * @var EventDispatcherInterface
+     */
+    protected $eventDispatcher;
+
+    /**
+     * @var string
+     */
+    protected $screenshotDir = '';
+
+    /**
+     * @var int
+     */
+    protected $screenshotWidth = 3840;
+
+    public function __construct(
+        EventDispatcherInterface $eventDispatcher,
+        string $screenshotDir,
+        int $screenshotWidth
+    ) {
+        $this->eventDispatcher = $eventDispatcher;
+        $this->screenshotDir = $this->convertReltiveFolder($screenshotDir);
+        $this->screenshotWidth = $screenshotWidth;
+    }
+
+    public function createScreenshot(string $url, int $height): string
+    {
+        // TODO: Include width in screenshot dir.
+        // This enables to compare different resolutions
+        $screenshotTarget = $this->getScreenshotTarget($url);
+        $this->createDir($this->screenshotDir . dirname($screenshotTarget));
+        $completeScreenshotTarget = $this->screenshotDir . $screenshotTarget;
+
+        $screenshotProcess = new Process([
+            'chromium-browser',
+            '--headless',
+            '--disable-gpu',
+            '--window-size=' . $this->screenshotWidth . ',' . $height,
+            '--screenshot=' . $completeScreenshotTarget,
+            $url
+        ]);
+        $screenshotProcess->setTimeout(60 * 2);
+        // TODO: Check for success
+        $screenshotProcess->run();
+
+        $this->eventDispatcher->dispatch(
+            'service.screenshot.created',
+            new GenericEvent('Created Screenshot', [
+                'screenshot' => $completeScreenshotTarget,
+                'url' => $url,
+            ])
+        );
+
+        return $completeScreenshotTarget;
+    }
+
+    public function getScreenshotTarget(string $url, string $suffix = 'png'): string
+    {
+        $uri = new Uri($url);
+
+        return implode(
+            DIRECTORY_SEPARATOR,
+            array_filter(
+                [
+                    $uri->getScheme(),
+                    $uri->getHost(),
+                    trim($uri->getPath(), '/'),
+                    $uri->getQuery(),
+                ],
+                function (string $string) {
+                    return trim($string, ' /') !== '';
+                }
+            )
+        ) . '.' . $suffix;
+    }
+
+    public function getScreenshotDir(): string
+    {
+        return $this->screenshotDir;
+    }
+
+    /**
+     * @throws \Exception If folder could not be created.
+     */
+    public function createDir(string $dir)
+    {
+        if (!is_dir($dir)) {
+            mkdir($dir, 0777, true);
+        }
+
+        if (!is_dir($dir)) {
+            throw new \Exception('Could not create directory: "' . $dir . '".', 1535528875);
+        }
+    }
+
+    public function convertReltiveFolder(string $folder): string
+    {
+        return implode(DIRECTORY_SEPARATOR, [
+            dirname(dirname(dirname(dirname(__FILE__)))),
+            rtrim($folder, '/'),
+        ]) . DIRECTORY_SEPARATOR;
+    }
+}