ScrapeRevisitUrlsCommand::findScrapers() - Code Metrics - treehouselabs/io-bundle - Measure and Improve Code Quality continuously with Scrutinizer

ScrapeRevisitUrlsCommand::findScrapers() A
last analyzed 2020-05-04 12:22 UTC

↳ Parent: ScrapeRevisitUrlsCommand

Complexity

Conditions	2
Paths	2

Size

Total Lines

Duplication

Lines	10
Ratio	100 %

Code Coverage

Tests	0
CRAP Score	6

Importance

Changes

Metric	Value
dl	10
loc	10
ccs	0
cts	8
cp	0
rs	9.9332
c	0
b	0
f	0
cc	2
nc	2
nop	1
crap	6

<?php

namespace TreeHouse\IoBundle\Command;

use Doctrine\Common\Persistence\ManagerRegistry;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use TreeHouse\IoBundle\Entity\Scraper;
use TreeHouse\IoBundle\Entity\Scraper as ScraperEntity;
use TreeHouse\IoBundle\Model\SourceInterface;
use TreeHouse\IoBundle\Scrape\Exception\CrawlException;
use TreeHouse\IoBundle\Scrape\SourceRevisitor;
use TreeHouse\IoBundle\Source\SourceManagerInterface;

class ScrapeRevisitUrlsCommand extends Command
{
    /**
     * @var ManagerRegistry
     */
    protected $doctrine;

    /**
     * @var SourceManagerInterface
     */
    protected $sourceManager;

    /**
     * @var SourceRevisitor
     */
    protected $revisitor;

    /**
     * @param ManagerRegistry        $doctrine
     * @param SourceManagerInterface $sourceManager
     * @param SourceRevisitor        $revisitor
     */
    public function __construct(ManagerRegistry $doctrine, SourceManagerInterface $sourceManager, SourceRevisitor $revisitor)

    {
        $this->doctrine = $doctrine;
        $this->sourceManager = $sourceManager;
        $this->revisitor = $revisitor;

        parent::__construct();
    }

    /**
     * @inheritdoc
     */
    protected function configure()

    {
        $this->setName('io:scrape:revisit-urls');
        $this->setDescription('Revisits earlier scraped sources to see if they still exist');
        $this->addOption('scraper', InputOption::VALUE_OPTIONAL, 'Limit the sources to a specific scraper id');
        $this->addOption(
            'async',
            'a',
            InputOption::VALUE_NONE,
            'Whether to revisit asynchronous. Doing so will queue sources, rather them revisiting them right away'
        );
        $this->addOption('no-limit', null, InputOption::VALUE_NONE, 'Disables the rate limit');
    }

    /**
     * @inheritdoc
     */
    protected function execute(InputInterface $input, OutputInterface $output)
    {
        $async = $input->getOption('async');
        $noLimit = $input->getOption('no-limit');
        $scrapers = $this->findScrapers($input->getArgument('scraper'));
function acceptsInteger($int) { }

$x = '123'; // string "123"

// Instead of
acceptsInteger($x);

// we recommend to use
acceptsInteger((integer) $x);

        foreach ($scrapers as $scraperEntity) {
            $date = new \DateTime(sprintf('-%d hours', $scraperEntity->getRevisitFrequency()));

            $builder = $this->sourceManager->getRepository()->queryByScraperAndUnvisitedSince($scraperEntity, $date);
            foreach ($builder->getQuery()->iterate() as list($source)) {
                /* @var SourceInterface $source */
                try {
                    $output->writeln(sprintf('Revisiting <info>%s</info>', $source->getOriginalUrl()));

                    $this->revisitor->revisit($source, $async, $noLimit);
                } catch (CrawlException $e) {
                    $output->writeln(sprintf('<error>%s</error>', $e->getMessage()));
                }
            }
        }
    }

    /**
     * @param integer[] $ids
     *
     * @return ScraperEntity[]
     */
    protected function findScrapers(array $ids)

    {
        $repo = $this->doctrine->getRepository('TreeHouseIoBundle:Scraper');

        if (!empty($ids)) {
            return $repo->findBy(['id' => $ids]);
        }

        return $repo->findAll();
    }
}


1		<?php
2
3		namespace TreeHouse\IoBundle\Command;
4
5		use Doctrine\Common\Persistence\ManagerRegistry;
6		use Symfony\Component\Console\Command\Command;
7		use Symfony\Component\Console\Input\InputInterface;
8		use Symfony\Component\Console\Input\InputOption;
9		use Symfony\Component\Console\Output\OutputInterface;
10		use TreeHouse\IoBundle\Entity\Scraper;
11		use TreeHouse\IoBundle\Entity\Scraper as ScraperEntity;
12		use TreeHouse\IoBundle\Model\SourceInterface;
13		use TreeHouse\IoBundle\Scrape\Exception\CrawlException;
14		use TreeHouse\IoBundle\Scrape\SourceRevisitor;
15		use TreeHouse\IoBundle\Source\SourceManagerInterface;
16
17		class ScrapeRevisitUrlsCommand extends Command
18		{
19		/**
20		* @var ManagerRegistry
21		*/
22		protected $doctrine;
23
24		/**
25		* @var SourceManagerInterface
26		*/
27		protected $sourceManager;
28
29		/**
30		* @var SourceRevisitor
31		*/
32		protected $revisitor;
33
34		/**
35		* @param ManagerRegistry $doctrine
36		* @param SourceManagerInterface $sourceManager
37		* @param SourceRevisitor $revisitor
38		*/
39	View Code Duplication	public function __construct(ManagerRegistry $doctrine, SourceManagerInterface $sourceManager, SourceRevisitor $revisitor)
		0 ignored issues – show Duplication introduced 2016-01-05 18:41 UTC by Report Bug Copy Issue Report This method seems to be duplicated in your project. Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation. You can also find more detailed suggestions in the “Code” section of your repository. Loading history...
40		{
41		$this->doctrine = $doctrine;
42		$this->sourceManager = $sourceManager;
43		$this->revisitor = $revisitor;
44
45		parent::__construct();
46		}
47
48		/**
49		* @inheritdoc
50		*/
51	View Code Duplication	protected function configure()
		0 ignored issues – show Duplication introduced 2017-10-04 10:43 UTC by Report Bug Copy Issue Report This method seems to be duplicated in your project. Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation. You can also find more detailed suggestions in the “Code” section of your repository. Loading history...
52		{
53		$this->setName('io:scrape:revisit-urls');
54		$this->setDescription('Revisits earlier scraped sources to see if they still exist');
55		$this->addOption('scraper', InputOption::VALUE_OPTIONAL, 'Limit the sources to a specific scraper id');
56		$this->addOption(
57		'async',
58		'a',
59		InputOption::VALUE_NONE,
60		'Whether to revisit asynchronous. Doing so will queue sources, rather them revisiting them right away'
61		);
62		$this->addOption('no-limit', null, InputOption::VALUE_NONE, 'Disables the rate limit');
63		}
64
65		/**
66		* @inheritdoc
67		*/
68		protected function execute(InputInterface $input, OutputInterface $output)
69		{
70		$async = $input->getOption('async');
71		$noLimit = $input->getOption('no-limit');
72		$scrapers = $this->findScrapers($input->getArgument('scraper'));
		0 ignored issues – show Documentation introduced 2018-12-12 10:10 UTC by Report Bug Copy Issue Report `$input->getArgument('scraper')` is of type `string\|array<integer,string>\|null`, but the function expects a `array<integer,integer>`. It seems like the type of the argument is not accepted by the function/method which you are calling. In some cases, in particular if PHP’s automatic type-juggling kicks in this might be fine. In other cases, however this might be a bug. We suggest to add an explicit type cast like in the following example: function acceptsInteger($int) { } $x = '123'; // string "123" // Instead of acceptsInteger($x); // we recommend to use acceptsInteger((integer) $x); Loading history...
73
74		foreach ($scrapers as $scraperEntity) {
75		$date = new \DateTime(sprintf('-%d hours', $scraperEntity->getRevisitFrequency()));
76
77		$builder = $this->sourceManager->getRepository()->queryByScraperAndUnvisitedSince($scraperEntity, $date);
78		foreach ($builder->getQuery()->iterate() as list($source)) {
79		/* @var SourceInterface $source */
80		try {
81		$output->writeln(sprintf('Revisiting <info>%s</info>', $source->getOriginalUrl()));
82
83		$this->revisitor->revisit($source, $async, $noLimit);
84		} catch (CrawlException $e) {
85		$output->writeln(sprintf('<error>%s</error>', $e->getMessage()));
86		}
87		}
88		}
89		}
90
91		/**
92		* @param integer[] $ids
93		*
94		* @return ScraperEntity[]
95		*/
96	View Code Duplication	protected function findScrapers(array $ids)
		0 ignored issues – show Duplication introduced 2017-10-04 10:43 UTC by Report Bug Copy Issue Report This method seems to be duplicated in your project. Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation. You can also find more detailed suggestions in the “Code” section of your repository. Loading history...
97		{
98		$repo = $this->doctrine->getRepository('TreeHouseIoBundle:Scraper');
99
100		if (!empty($ids)) {
101		return $repo->findBy(['id' => $ids]);
102		}
103
104		return $repo->findAll();
105		}
106		}
107

treehouselabs / io-bundle

ScrapeRevisitUrlsCommand::findScrapers() A last analyzed 2020-05-04 12:22 UTC

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like

ScrapeRevisitUrlsCommand::findScrapers() A
last analyzed 2020-05-04 12:22 UTC