From 09bdc7e73f88702939fef515c6719723fbf80764 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Mon, 3 Sep 2018 16:13:47 +0200 Subject: [PATCH] TASK: Add recover file --- comparison | 7 +++ src/Command/CompareCommand.php | 48 +++++++++++++++++++-- src/Command/CreateBaseCommand.php | 52 +++++++++++++++++++++-- src/Model/UrlListDto.php | 37 ++++++++++++---- src/Model/UrlListDtoFactory.php | 48 +++++++++++++++++++++ src/Service/Screenshot/CrawlerService.php | 5 +-- src/Service/Screenshot/Service.php | 5 ++- 7 files changed, 180 insertions(+), 22 deletions(-) create mode 100644 src/Model/UrlListDtoFactory.php diff --git a/comparison b/comparison index ae55bbb..85419c8 100755 --- a/comparison +++ b/comparison @@ -1,6 +1,12 @@ #!/usr/bin/env php setDispatcher($eventDispatcher); diff --git a/src/Command/CompareCommand.php b/src/Command/CompareCommand.php index 69638a5..c0ea5d9 100644 --- a/src/Command/CompareCommand.php +++ b/src/Command/CompareCommand.php @@ -21,6 +21,8 @@ namespace Codappix\WebsiteComparison\Command; * 02110-1301, USA. */ +use Codappix\WebsiteComparison\Model\UrlListDto; +use Codappix\WebsiteComparison\Model\UrlListDtoFactory; use Codappix\WebsiteComparison\Service\Screenshot\CompareService; use Codappix\WebsiteComparison\Service\Screenshot\CrawlerService; use Codappix\WebsiteComparison\Service\Screenshot\Service; @@ -90,6 +92,13 @@ class CompareCommand extends Command 'The width for screen resolution and screenshots.', 3840 ) + ->addOption( + 'recoverFile', + null, + InputOption::VALUE_OPTIONAL, + 'Path to json-File with state of stopped process, used to recover process.', + '' + ) ->addArgument( 'baseUrl', @@ -101,6 +110,7 @@ class CompareCommand extends Command protected function execute(InputInterface $input, OutputInterface $output) { + $baseUrl = $input->getArgument('baseUrl'); $screenshotService = new Service( $this->eventDispatcher, @@ -111,7 +121,7 @@ class CompareCommand extends Command $screenshotCrawler = new CrawlerService( $this->webDriver, $screenshotService, - $input->getArgument('baseUrl') + $baseUrl ); $compareService = new CompareService( @@ -120,15 +130,47 @@ class CompareCommand extends Command $input->getOption('screenshotDir'), $input->getOption('diffResultDir') ); - $this->registerEvents($output, $compareService); - $screenshotCrawler->crawl(); + $linkList = $this->getLinkList($baseUrl, $input->getOption('recoverFile')); + + $this->registerEvents($output, $compareService); + try { + $screenshotCrawler->crawl($linkList); + } catch (\Exception $e) { + file_put_contents($this->getJsonFilePath($screenshotService, $baseUrl), json_encode($linkList)); + $output->writeln(sprintf( + 'Saved current state for recovering in "%s".', + $this->getJsonFilePath($screenshotService, $baseUrl) + )); + throw $e; + } if ($compareService->hasDifferences()) { return 255; } } + protected function getLinkList( + string $baseUrl, + string $recoverFile = '' + ): UrlListDto { + $factory = new UrlListDtoFactory(); + + if (trim($recoverFile) !== '') { + return $factory->createWithByConfigurationFile($recoverFile); + } + + return $factory->createWithBaseUrl($baseUrl); + } + + protected function getJsonFilePath(Service $screenshotService, string $baseUrl): string + { + return $screenshotService->getScreenshotDir() . + DIRECTORY_SEPARATOR . + $screenshotService->getScreenshotTarget($baseUrl, 'json') + ; + } + protected function registerEvents(OutputInterface $output, CompareService $compareService) { $this->eventDispatcher->addListener( diff --git a/src/Command/CreateBaseCommand.php b/src/Command/CreateBaseCommand.php index 4bf4d57..8b8de9e 100644 --- a/src/Command/CreateBaseCommand.php +++ b/src/Command/CreateBaseCommand.php @@ -21,6 +21,8 @@ namespace Codappix\WebsiteComparison\Command; * 02110-1301, USA. */ +use Codappix\WebsiteComparison\Model\UrlListDto; +use Codappix\WebsiteComparison\Model\UrlListDtoFactory; use Codappix\WebsiteComparison\Service\Screenshot\CrawlerService; use Codappix\WebsiteComparison\Service\Screenshot\Service; use Facebook\WebDriver\Remote\RemoteWebDriver; @@ -75,6 +77,13 @@ class CreateBaseCommand extends Command 'The width for screen resolution and screenshots.', 3840 ) + ->addOption( + 'recoverFile', + null, + InputOption::VALUE_OPTIONAL, + 'Path to json-File with state of stopped process, used to recover process.', + '' + ) ->addArgument( 'baseUrl', @@ -86,20 +95,55 @@ class CreateBaseCommand extends Command protected function execute(InputInterface $input, OutputInterface $output) { - $this->registerEvents($output); + $baseUrl = $input->getArgument('baseUrl'); + $screenshotDir = $input->getOption('screenshotDir'); $screenshotService = new Service( $this->eventDispatcher, - $input->getOption('screenshotDir'), + $screenshotDir, $input->getOption('screenshotWidth') ); $screenshotCrawler = new CrawlerService( $this->webDriver, $screenshotService, - $input->getArgument('baseUrl') + $baseUrl ); - $screenshotCrawler->crawl(); + + $linkList = $this->getLinkList($baseUrl, $input->getOption('recoverFile')); + + $this->registerEvents($output); + try { + $screenshotCrawler->crawl($linkList); + } catch (\Exception $e) { + file_put_contents($this->getJsonFilePath($screenshotService, $baseUrl), json_encode($linkList)); + $output->writeln(sprintf( + 'Saved current state for recovering in "%s".', + $this->getJsonFilePath($screenshotService, $baseUrl) + )); + throw $e; + } + } + + protected function getLinkList( + string $baseUrl, + string $recoverFile = '' + ): UrlListDto { + $factory = new UrlListDtoFactory(); + + if (trim($recoverFile) !== '') { + return $factory->createWithByConfigurationFile($recoverFile); + } + + return $factory->createWithBaseUrl($baseUrl); + } + + protected function getJsonFilePath(Service $screenshotService, string $baseUrl): string + { + return $screenshotService->getScreenshotDir() . + DIRECTORY_SEPARATOR . + $screenshotService->getScreenshotTarget($baseUrl, 'json') + ; } protected function registerEvents(OutputInterface $output) diff --git a/src/Model/UrlListDto.php b/src/Model/UrlListDto.php index 0573968..295028b 100644 --- a/src/Model/UrlListDto.php +++ b/src/Model/UrlListDto.php @@ -22,21 +22,32 @@ namespace Codappix\WebsiteComparison\Model; */ /** + * List of urls with two states. * + * Allows to have a single queue of urls to work on. */ -class UrlListDto +class UrlListDto implements \JsonSerializable { protected $finishedUrls = []; protected $upcomingUrls = []; - public function addUrl(string $link) + public function addUrl(string $url) { - if ($this->isUrlKnown($link)) { + if ($this->isUrlKnown($url)) { return; } - $this->upcomingUrls[] = $link; + $this->upcomingUrls[] = $url; + } + + public function addFinishedUrl(string $url) + { + if ($this->isUrlKnown($url)) { + return; + } + + $this->finishedUrls[] = $url; } public function getNextUrl(): string @@ -44,17 +55,25 @@ class UrlListDto return reset($this->upcomingUrls) ?? ''; } - public function markUrlAsFinished(string $link) + public function markUrlAsFinished(string $url) { - $upcomingEntry = array_search($link, $this->upcomingUrls); + $upcomingEntry = array_search($url, $this->upcomingUrls); unset($this->upcomingUrls[$upcomingEntry]); - $this->finishedUrls[] = $link; + $this->finishedUrls[] = $url; } - public function isUrlKnown(string $link): bool + public function isUrlKnown(string $url): bool { - return in_array($link, $this->finishedUrls) || in_array($link, $this->upcomingUrls); + return in_array($url, $this->finishedUrls) || in_array($url, $this->upcomingUrls); + } + + public function jsonSerialize() + { + return [ + 'finishedUrls' => $this->finishedUrls, + 'upcomingUrls' => $this->upcomingUrls, + ]; } } diff --git a/src/Model/UrlListDtoFactory.php b/src/Model/UrlListDtoFactory.php new file mode 100644 index 0000000..b8d9a5e --- /dev/null +++ b/src/Model/UrlListDtoFactory.php @@ -0,0 +1,48 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +class UrlListDtoFactory +{ + public function createWithBaseUrl(string $baseUrl): UrlListDto + { + $urlList = new UrlListDto(); + $urlList->addUrl($baseUrl); + + return $urlList; + } + + public function createWithByConfigurationFile(string $configFile): UrlListDto + { + $urlList = new UrlListDto(); + $config = json_decode(file_get_contents($configFile), true); + + foreach ($config['upcomingUrls'] as $url) { + $urlList->addUrl($url); + } + foreach ($config['finishedUrls'] as $url) { + $urlList->addFinishedUrl($url); + } + + return $urlList; + } +} diff --git a/src/Service/Screenshot/CrawlerService.php b/src/Service/Screenshot/CrawlerService.php index 629ae08..7854102 100644 --- a/src/Service/Screenshot/CrawlerService.php +++ b/src/Service/Screenshot/CrawlerService.php @@ -54,11 +54,8 @@ class CrawlerService $this->baseUrl = rtrim($baseUrl, '/') . '/'; } - public function crawl() + public function crawl(UrlListDto $linkList) { - $linkList = new UrlListDto(); - $linkList->addUrl($this->baseUrl); - while ($url = $linkList->getNextUrl()) { $this->driver->get($url); $screenshotHeight = $this->driver->findElement(WebDriverBy::cssSelector('body')) diff --git a/src/Service/Screenshot/Service.php b/src/Service/Screenshot/Service.php index e8d1f28..6fdd49c 100644 --- a/src/Service/Screenshot/Service.php +++ b/src/Service/Screenshot/Service.php @@ -69,6 +69,7 @@ class Service '--screenshot=' . $completeScreenshotTarget, $url ]); + $screenshotProcess->setTimeout(60 * 2); // TODO: Check for success $screenshotProcess->run(); @@ -83,7 +84,7 @@ class Service return $completeScreenshotTarget; } - protected function getScreenshotTarget(string $url): string + public function getScreenshotTarget(string $url, string $suffix = 'png'): string { $uri = new Uri($url); @@ -100,7 +101,7 @@ class Service return trim($string, ' /') !== ''; } ) - ) . '.png'; + ) . '.' . $suffix; } public function getScreenshotDir(): string