TASK: Add recover file

comparison

@@ -1,6 +1,12 @@
 #!/usr/bin/env php
 <?php
 
+declare(ticks = 1);
+
+pcntl_signal(SIGINT, function () {
+    exit(99);
+});
+
 require __DIR__ . '/vendor/autoload.php';
 
 use Codappix\WebsiteComparison\Command\CompareCommand;
@@ -25,6 +31,7 @@ $chromeDriver = (function () {
     return ChromeDriver::start(null, $chromeDriverService);
 })();
 
+
 $application = new Application();
 $application->setDispatcher($eventDispatcher);
 

src/Command/CompareCommand.php

@@ -21,6 +21,8 @@ namespace Codappix\WebsiteComparison\Command;
  * 02110-1301, USA.
  */
 
+use Codappix\WebsiteComparison\Model\UrlListDto;
+use Codappix\WebsiteComparison\Model\UrlListDtoFactory;
 use Codappix\WebsiteComparison\Service\Screenshot\CompareService;
 use Codappix\WebsiteComparison\Service\Screenshot\CrawlerService;
 use Codappix\WebsiteComparison\Service\Screenshot\Service;
@@ -90,6 +92,13 @@ class CompareCommand extends Command
                 'The width for screen resolution and screenshots.',
                 3840
             )
+            ->addOption(
+                'recoverFile',
+                null,
+                InputOption::VALUE_OPTIONAL,
+                'Path to json-File with state of stopped process, used to recover process.',
+                ''
+            )
 
             ->addArgument(
                 'baseUrl',
@@ -101,6 +110,7 @@ class CompareCommand extends Command
 
     protected function execute(InputInterface $input, OutputInterface $output)
     {
+        $baseUrl = $input->getArgument('baseUrl');
 
         $screenshotService = new Service(
             $this->eventDispatcher,
@@ -111,7 +121,7 @@ class CompareCommand extends Command
         $screenshotCrawler = new CrawlerService(
             $this->webDriver,
             $screenshotService,
-            $input->getArgument('baseUrl')
+            $baseUrl
         );
 
         $compareService = new CompareService(
@@ -120,15 +130,47 @@ class CompareCommand extends Command
             $input->getOption('screenshotDir'),
             $input->getOption('diffResultDir')
         );
-        $this->registerEvents($output, $compareService);
 
-        $screenshotCrawler->crawl();
+        $linkList = $this->getLinkList($baseUrl, $input->getOption('recoverFile'));
+
+        $this->registerEvents($output, $compareService);
+        try {
+            $screenshotCrawler->crawl($linkList);
+        } catch (\Exception $e) {
+            file_put_contents($this->getJsonFilePath($screenshotService, $baseUrl), json_encode($linkList));
+            $output->writeln(sprintf(
+                '<comment>Saved current state for recovering in "%s".</comment>',
+                $this->getJsonFilePath($screenshotService, $baseUrl)
+            ));
+            throw $e;
+        }
 
         if ($compareService->hasDifferences()) {
             return 255;
         }
     }
 
+    protected function getLinkList(
+        string $baseUrl,
+        string $recoverFile = ''
+    ): UrlListDto {
+        $factory = new UrlListDtoFactory();
+
+        if (trim($recoverFile) !== '') {
+            return $factory->createWithByConfigurationFile($recoverFile);
+        }
+
+        return $factory->createWithBaseUrl($baseUrl);
+    }
+
+    protected function getJsonFilePath(Service $screenshotService, string $baseUrl): string
+    {
+        return $screenshotService->getScreenshotDir() .
+            DIRECTORY_SEPARATOR .
+            $screenshotService->getScreenshotTarget($baseUrl, 'json')
+            ;
+    }
+
     protected function registerEvents(OutputInterface $output, CompareService $compareService)
     {
         $this->eventDispatcher->addListener(

src/Command/CreateBaseCommand.php

@@ -21,6 +21,8 @@ namespace Codappix\WebsiteComparison\Command;
  * 02110-1301, USA.
  */
 
+use Codappix\WebsiteComparison\Model\UrlListDto;
+use Codappix\WebsiteComparison\Model\UrlListDtoFactory;
 use Codappix\WebsiteComparison\Service\Screenshot\CrawlerService;
 use Codappix\WebsiteComparison\Service\Screenshot\Service;
 use Facebook\WebDriver\Remote\RemoteWebDriver;
@@ -75,6 +77,13 @@ class CreateBaseCommand extends Command
                 'The width for screen resolution and screenshots.',
                 3840
             )
+            ->addOption(
+                'recoverFile',
+                null,
+                InputOption::VALUE_OPTIONAL,
+                'Path to json-File with state of stopped process, used to recover process.',
+                ''
+            )
 
             ->addArgument(
                 'baseUrl',
@@ -86,20 +95,55 @@ class CreateBaseCommand extends Command
 
     protected function execute(InputInterface $input, OutputInterface $output)
     {
-        $this->registerEvents($output);
+        $baseUrl = $input->getArgument('baseUrl');
+        $screenshotDir = $input->getOption('screenshotDir');
 
         $screenshotService = new Service(
             $this->eventDispatcher,
-            $input->getOption('screenshotDir'),
+            $screenshotDir,
             $input->getOption('screenshotWidth')
         );
 
         $screenshotCrawler = new CrawlerService(
             $this->webDriver,
             $screenshotService,
-            $input->getArgument('baseUrl')
+            $baseUrl
         );
-        $screenshotCrawler->crawl();
+
+        $linkList = $this->getLinkList($baseUrl, $input->getOption('recoverFile'));
+
+        $this->registerEvents($output);
+        try {
+            $screenshotCrawler->crawl($linkList);
+        } catch (\Exception $e) {
+            file_put_contents($this->getJsonFilePath($screenshotService, $baseUrl), json_encode($linkList));
+            $output->writeln(sprintf(
+                '<comment>Saved current state for recovering in "%s".</comment>',
+                $this->getJsonFilePath($screenshotService, $baseUrl)
+            ));
+            throw $e;
+        }
+    }
+
+    protected function getLinkList(
+        string $baseUrl,
+        string $recoverFile = ''
+    ): UrlListDto {
+        $factory = new UrlListDtoFactory();
+
+        if (trim($recoverFile) !== '') {
+            return $factory->createWithByConfigurationFile($recoverFile);
+        }
+
+        return $factory->createWithBaseUrl($baseUrl);
+    }
+
+    protected function getJsonFilePath(Service $screenshotService, string $baseUrl): string
+    {
+        return $screenshotService->getScreenshotDir() .
+            DIRECTORY_SEPARATOR .
+            $screenshotService->getScreenshotTarget($baseUrl, 'json')
+            ;
     }
 
     protected function registerEvents(OutputInterface $output)

src/Model/UrlListDto.php

@@ -22,21 +22,32 @@ namespace Codappix\WebsiteComparison\Model;
  */
 
 /**
+ * List of urls with two states.
  *
+ * Allows to have a single queue of urls to work on.
  */
-class UrlListDto
+class UrlListDto implements \JsonSerializable
 {
     protected $finishedUrls = [];
 
     protected $upcomingUrls = [];
 
-    public function addUrl(string $link)
+    public function addUrl(string $url)
     {
-        if ($this->isUrlKnown($link)) {
+        if ($this->isUrlKnown($url)) {
             return;
         }
 
-        $this->upcomingUrls[] = $link;
+        $this->upcomingUrls[] = $url;
+    }
+
+    public function addFinishedUrl(string $url)
+    {
+        if ($this->isUrlKnown($url)) {
+            return;
+        }
+
+        $this->finishedUrls[] = $url;
     }
 
     public function getNextUrl(): string
@@ -44,17 +55,25 @@ class UrlListDto
         return reset($this->upcomingUrls) ?? '';
     }
 
-    public function markUrlAsFinished(string $link)
+    public function markUrlAsFinished(string $url)
     {
-        $upcomingEntry = array_search($link, $this->upcomingUrls);
+        $upcomingEntry = array_search($url, $this->upcomingUrls);
 
         unset($this->upcomingUrls[$upcomingEntry]);
 
-        $this->finishedUrls[] = $link;
+        $this->finishedUrls[] = $url;
     }
 
-    public function isUrlKnown(string $link): bool
+    public function isUrlKnown(string $url): bool
     {
-        return in_array($link, $this->finishedUrls) || in_array($link, $this->upcomingUrls);
+        return in_array($url, $this->finishedUrls) || in_array($url, $this->upcomingUrls);
+    }
+
+    public function jsonSerialize()
+    {
+        return [
+            'finishedUrls' => $this->finishedUrls,
+            'upcomingUrls' => $this->upcomingUrls,
+        ];
     }
 }

src/Model/UrlListDtoFactory.php

@@ -0,0 +1,48 @@
+<?php
+
+namespace Codappix\WebsiteComparison\Model;
+
+/*
+ * Copyright (C) 2018 Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+class UrlListDtoFactory
+{
+    public function createWithBaseUrl(string $baseUrl): UrlListDto
+    {
+        $urlList = new UrlListDto();
+        $urlList->addUrl($baseUrl);
+
+        return $urlList;
+    }
+
+    public function createWithByConfigurationFile(string $configFile): UrlListDto
+    {
+        $urlList = new UrlListDto();
+        $config = json_decode(file_get_contents($configFile), true);
+
+        foreach ($config['upcomingUrls'] as $url) {
+            $urlList->addUrl($url);
+        }
+        foreach ($config['finishedUrls'] as $url) {
+            $urlList->addFinishedUrl($url);
+        }
+
+        return $urlList;
+    }
+}

src/Service/Screenshot/CrawlerService.php

@@ -54,11 +54,8 @@ class CrawlerService
         $this->baseUrl = rtrim($baseUrl, '/') . '/';
     }
 
-    public function crawl()
+    public function crawl(UrlListDto $linkList)
     {
-        $linkList = new UrlListDto();
-        $linkList->addUrl($this->baseUrl);
-
         while ($url = $linkList->getNextUrl()) {
             $this->driver->get($url);
             $screenshotHeight = $this->driver->findElement(WebDriverBy::cssSelector('body'))

src/Service/Screenshot/Service.php

@@ -69,6 +69,7 @@ class Service
             '--screenshot=' . $completeScreenshotTarget,
             $url
         ]);
+        $screenshotProcess->setTimeout(60 * 2);
         // TODO: Check for success
         $screenshotProcess->run();
 
@@ -83,7 +84,7 @@ class Service
         return $completeScreenshotTarget;
     }
 
-    protected function getScreenshotTarget(string $url): string
+    public function getScreenshotTarget(string $url, string $suffix = 'png'): string
     {
         $uri = new Uri($url);
 
@@ -100,7 +101,7 @@ class Service
                     return trim($string, ' /') !== '';
                 }
             )
-        ) . '.png';
+        ) . '.' . $suffix;
     }
 
     public function getScreenshotDir(): string