runProcesses() - Code Metrics - Inspection of "Add reports" - cimmwolf/sitemap-parser - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — master ( 9e19cf...b74b72 )

by Denis

created 2018-01-30 08:09 UTC

runProcesses() B

↳ Parent: Project

Complexity

Conditions	6
Paths	12

Size

Total Lines	45
Code Lines	34

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
eloc	34
nc	12
nop	3
dl	0
loc	45
c	0
b	0
f	0
cc	6
rs	8.439

<?php
require_once __DIR__ . '/config.php';

use Symfony\Component\Console\Helper\ProgressBar;
use Symfony\Component\Console\Output\OutputInterface;

define('PID', getmypid());
/**
 * @param array $pages
 * @param string $task
 * @param $output OutputInterface
 */
function runProcesses($pages, $task, &$output)
{
    $pdo = new PDO(DSN);
    $pdo->query('DROP TABLE IF EXISTS pages');
    $pdo->query('CREATE TABLE pages (url TEXT NOT NULL, status TEXT, scope INTEGER NOT NULL)');

    for ($i = 0; $i < PROCESSES; $i++) {
        $sliceSize = ceil(count($pages) / PROCESSES);
        $slice = array_slice($pages, $i * $sliceSize, $sliceSize);

        foreach (array_chunk($slice, 999) as $part) {
            $values = implode(',', array_pad([], count($part), "(?,$i)"));
            $stmt = $pdo->prepare("INSERT INTO pages (url, scope) VALUES $values");
            if (!$stmt) {
                fwrite(STDERR, $pdo->errorInfo()[2] . PHP_EOL);
            }
            $stmt->execute($part);
            unset($stmt);
        }

        $cPath = __DIR__ . '/_process.php';
        execInBackground("php $cPath -s $i -p " . PID . " -t $task");
    }

    $do = count($pages);
    $done = 0;
    $progress = new ProgressBar($output, $do);
    $progress->setRedrawFrequency(10);
    $progress->setFormatDefinition('custom', ' %current%/%max% %bar% %message%');
    $progress->setFormat('custom');
    $progress->setMessage('');
    $progress->start();
    while ($done < $do) {
        $done = $pdo->query('SELECT COUNT(status) FROM pages WHERE status IS NOT NULL')->fetchColumn();
        $codes = $pdo->query('SELECT status, COUNT(status) AS count FROM pages WHERE status IS NOT NULL GROUP BY status')->fetchAll(PDO::FETCH_ASSOC);
        $message = [];
        foreach ($codes as $code) {
            $message[] = str_replace(200, 'ok', $code['status']) . ': ' . $code['count'];
        }
        $progress->setMessage(implode(' ', $message));
        $progress->setProgress($done);
        usleep(500000);
    }
    $progress->finish();
    $output->writeln('');
}

/**
 * @param $website_url
 * @param $output OutputInterface
 * @return array
 * @throws Exception
 */
function getPages($website_url, &$output)
{
    $output->writeln("  Getting sitemap of $website_url ...");
    $sitemap = simplexml_load_file($website_url . '/sitemap.xml');

    if ($sitemap === false) {
        throw new Exception("Can't get $website_url . /sitemap.xml");
    }

    $pages = [];
    foreach ($sitemap as $url) {
        $pages[] = urldecode($url->loc);
    }
    return $pages;
}

/**
 * @param $cmd string
 */
function execInBackground($cmd)
{
    if (substr(php_uname(), 0, 7) == "Windows") {
        pclose(popen("start /B " . $cmd, "r"));
    } else {
        exec($cmd . " > /dev/null 2>/dev/null &");
    }
}

/**
 * @param $domain
 * @param $output OutputInterface
 */
function saveCheckReport($domain, &$output)
{
    $output->write('  Saving report...');
    $result = new PDO(DSN);
    $report = new PDO('sqlite:' . __DIR__ . "/reports/{$domain}.sqlite3");
    $report->query('DROP TABLE IF EXISTS `check`');
    $report->query('CREATE TABLE `check` (url TEXT NOT NULL, status TEXT NOT NULL)');
    $stm = $report->prepare('INSERT INTO `check` (url, status) VALUES (:url, :status)');
    $checks = $result->query('SELECT url, status FROM pages')->fetchAll(PDO::FETCH_ASSOC);
    foreach ($checks as $check) {
        $stm->execute($check);
    }
    $output->writeln('  ok');
}

/**
 * @param $domain
 * @param $output OutputInterface
 */
function saveLinksReport($domain, &$output)
{
    $output->write('  Saving report...');
    $result = new PDO(DSN);
    $report = new PDO('sqlite:' . __DIR__ . "/reports/{$domain}.sqlite3");
    $report->query('DROP TABLE IF EXISTS links');
    $report->query('CREATE TABLE links (page TEXT NOT NULL, link TEXT NOT NULL, url TEXT NOT NULL, status TEXT NOT NULL )');
    $stm = $report->prepare('INSERT INTO links (page, link, url, status) VALUES (:page, :link, :url, :status)');
    $checks = $result->query('SELECT url, status FROM pages')->fetchAll();
    foreach ($checks as $check) {
        $stm2 = $result->prepare('SELECT page, link FROM links WHERE url=:url');
        $stm2->execute([':url' => $check['url']]);
        $parameters = array_merge($check, $stm2->fetch(PDO::FETCH_ASSOC));
        $stm->execute($parameters);
    }
    $output->writeln('  ok');
}

if (!file_exists(__DIR__ . '/reports')) {
    mkdir(__DIR__ . '/reports');
}

$app = new Silly\Application();

$app->command('check website_url', function ($website_url, OutputInterface $output) {
    try {
        $pages = getPages($website_url, $output);
    } catch (Exception $exception) {
        $output->writeln($exception->getMessage());
        exit;

    }
    runProcesses($pages, 'check', $output);

    saveCheckReport(parse_url($website_url, PHP_URL_HOST), $output);
});

$app->command('links website_url', function ($website_url, OutputInterface $output) {
    try {
        $pages = getPages($website_url, $output);
    } catch (Exception $exception) {
        $output->writeln($exception->getMessage());
        exit;

    };

    $pdo = new PDO(DSN);
    $pdo->query('DROP TABLE IF EXISTS links');
    $pdo->query('CREATE TABLE links (page TEXT NOT NULL, link TEXT NOT NULL, url TEXT NOT NULL)');
    $pdo->query('CREATE UNIQUE INDEX links_url_uindex ON links (url);');
    unset($pdo);

    runProcesses($pages, 'parse-links', $output);

    $domain = parse_url($website_url, PHP_URL_HOST);
    saveCheckReport($domain, $output);

    $pdo = new PDO(DSN);
    $pages = $pdo->query('SELECT url FROM links')->fetchAll(PDO::FETCH_COLUMN);
    unset($pdo);

    $output->writeln('  Checking founded links...');

    runProcesses($pages, 'check', $output);

    saveLinksReport($domain, $output);
});

$app->command('metadata website_url', function ($website_url, OutputInterface $output) {
    /**
     * @param array $tree
     * @param callable $function
     * @param int $level
     * @param string $path
     */
    function walker($tree, $function, $level = 0, $path = '')
    {
        foreach ($tree as $branchName => $branch) {
            if (isset($branch['_self'])) {
                $function($branch['_self'], $level, $path);
                unset($branch['_self']);
            }
            if (count($branch) > 0)
                walker($branch, $function, $level + 1, "$path/$branchName");
        }
    }

    $output->writeln("  Getting sitemap of $website_url ...");
    $sitemap = simplexml_load_file($website_url . '/sitemap.xml');

    $resultFile = __DIR__ . '/runtime/' . parse_url($website_url, PHP_URL_HOST) . '-metadata.csv';

    $paths = [];
    foreach ($sitemap as $url) {
        $paths[] = parse_url($url->loc, PHP_URL_PATH);
    }
    natsort($paths);

    $tree = [];
    foreach ($paths as $path) {
        $levels = explode('/', $path);
        $temp = &$tree;
        foreach ($levels as $key => $level) {
            // в условии неочевидное преобразование для анализа ссылки на главную страницу
            if (!empty($level) || (empty(array_filter($levels)) && $level = '/')) {
                if (!isset($temp[$level]))
                    $temp[$level] = [];

                if ($key == (count($levels) - 1))
                    $temp[$level]['_self'] = ['path' => $path];

                $temp = &$temp[$level];
            }
        }
    }
    unset($temp);

    file_put_contents($resultFile, 'URL, Title, Keywords, Description, "Build Time: ' . date('r') . '"' . PHP_EOL);
    $previous = '';
    $progress = new \cli\progress\Bar(' Getting meta data', count($paths), 1000);
    walker($tree, function (&$self, $level, $path) use ($website_url, &$previous, $resultFile, &$progress) {
        $data = [];
        $page = file_get_contents($website_url . $self['path']);
        preg_match('~<title>(.*?)</title>~', $page, $temp);
        $data[] = $temp[1] ?? '';
        preg_match('~<meta name="keywords" content="(.*?)">~', $page, $temp);
        $data[] = $temp[1] ?? '';
        preg_match('~<meta name="description" content="(.*?)">~', $page, $temp);
        $data[] = $temp[1] ?? '';

        if ($data == $previous) {
            foreach ($data as &$item)
                $item = '--//--';
        } else
            $previous = $data;

        $row = [];
        $row[] = $website_url . $self['path'];
        $row = array_merge($row, $data);

        foreach ($row as &$item)
            $item = '"' . $item . '"';

        $line = implode(',', $row) . PHP_EOL;
        file_put_contents($resultFile, $line, FILE_APPEND);
        $progress->tick();
    });

    $progress->finish();
});

/** @noinspection PhpUnhandledExceptionInspection */
$app->run();

1			<?php
2			require_once __DIR__ . '/config.php';
3
4			use Symfony\Component\Console\Helper\ProgressBar;
5			use Symfony\Component\Console\Output\OutputInterface;
6
7			define('PID', getmypid());
8			/**
9			* @param array $pages
10			* @param string $task
11			* @param $output OutputInterface
12			*/
13			function runProcesses($pages, $task, &$output)
14			{
15			$pdo = new PDO(DSN);
16			$pdo->query('DROP TABLE IF EXISTS pages');
17			$pdo->query('CREATE TABLE pages (url TEXT NOT NULL, status TEXT, scope INTEGER NOT NULL)');
18
19			for ($i = 0; $i < PROCESSES; $i++) {
20			$sliceSize = ceil(count($pages) / PROCESSES);
21			$slice = array_slice($pages, $i * $sliceSize, $sliceSize);
22
23			foreach (array_chunk($slice, 999) as $part) {
24			$values = implode(',', array_pad([], count($part), "(?,$i)"));
25			$stmt = $pdo->prepare("INSERT INTO pages (url, scope) VALUES $values");
26			if (!$stmt) {
27			fwrite(STDERR, $pdo->errorInfo()[2] . PHP_EOL);
28			}
29			$stmt->execute($part);
30			unset($stmt);
31			}
32
33			$cPath = __DIR__ . '/_process.php';
34			execInBackground("php $cPath -s $i -p " . PID . " -t $task");
35			}
36
37			$do = count($pages);
38			$done = 0;
39			$progress = new ProgressBar($output, $do);
40			$progress->setRedrawFrequency(10);
41			$progress->setFormatDefinition('custom', ' %current%/%max% %bar% %message%');
42			$progress->setFormat('custom');
43			$progress->setMessage('');
44			$progress->start();
45			while ($done < $do) {
46			$done = $pdo->query('SELECT COUNT(status) FROM pages WHERE status IS NOT NULL')->fetchColumn();
47			$codes = $pdo->query('SELECT status, COUNT(status) AS count FROM pages WHERE status IS NOT NULL GROUP BY status')->fetchAll(PDO::FETCH_ASSOC);
48			$message = [];
49			foreach ($codes as $code) {
50			$message[] = str_replace(200, 'ok', $code['status']) . ': ' . $code['count'];
51			}
52			$progress->setMessage(implode(' ', $message));
53			$progress->setProgress($done);
54			usleep(500000);
55			}
56			$progress->finish();
57			$output->writeln('');
58			}
59
60			/**
61			* @param $website_url
62			* @param $output OutputInterface
63			* @return array
64			* @throws Exception
65			*/
66			function getPages($website_url, &$output)
67			{
68			$output->writeln(" Getting sitemap of $website_url ...");
69			$sitemap = simplexml_load_file($website_url . '/sitemap.xml');
70
71			if ($sitemap === false) {
72			throw new Exception("Can't get $website_url . /sitemap.xml");
73			}
74
75			$pages = [];
76			foreach ($sitemap as $url) {
77			$pages[] = urldecode($url->loc);
78			}
79			return $pages;
80			}
81
82			/**
83			* @param $cmd string
84			*/
85			function execInBackground($cmd)
86			{
87			if (substr(php_uname(), 0, 7) == "Windows") {
88			pclose(popen("start /B " . $cmd, "r"));
89			} else {
90			exec($cmd . " > /dev/null 2>/dev/null &");
91			}
92			}
93
94			/**
95			* @param $domain
96			* @param $output OutputInterface
97			*/
98			function saveCheckReport($domain, &$output)
99			{
100			$output->write(' Saving report...');
101			$result = new PDO(DSN);
102			$report = new PDO('sqlite:' . __DIR__ . "/reports/{$domain}.sqlite3");
103			$report->query('DROP TABLE IF EXISTS `check`');
104			$report->query('CREATE TABLE `check` (url TEXT NOT NULL, status TEXT NOT NULL)');
105			$stm = $report->prepare('INSERT INTO `check` (url, status) VALUES (:url, :status)');
106			$checks = $result->query('SELECT url, status FROM pages')->fetchAll(PDO::FETCH_ASSOC);
107			foreach ($checks as $check) {
108			$stm->execute($check);
109			}
110			$output->writeln(' ok');
111			}
112
113			/**
114			* @param $domain
115			* @param $output OutputInterface
116			*/
117			function saveLinksReport($domain, &$output)
118			{
119			$output->write(' Saving report...');
120			$result = new PDO(DSN);
121			$report = new PDO('sqlite:' . __DIR__ . "/reports/{$domain}.sqlite3");
122			$report->query('DROP TABLE IF EXISTS links');
123			$report->query('CREATE TABLE links (page TEXT NOT NULL, link TEXT NOT NULL, url TEXT NOT NULL, status TEXT NOT NULL )');
124			$stm = $report->prepare('INSERT INTO links (page, link, url, status) VALUES (:page, :link, :url, :status)');
125			$checks = $result->query('SELECT url, status FROM pages')->fetchAll();
126			foreach ($checks as $check) {
127			$stm2 = $result->prepare('SELECT page, link FROM links WHERE url=:url');
128			$stm2->execute([':url' => $check['url']]);
129			$parameters = array_merge($check, $stm2->fetch(PDO::FETCH_ASSOC));
130			$stm->execute($parameters);
131			}
132			$output->writeln(' ok');
133			}
134
135			if (!file_exists(__DIR__ . '/reports')) {
136			mkdir(__DIR__ . '/reports');
137			}
138
139			$app = new Silly\Application();
140
141			$app->command('check website_url', function ($website_url, OutputInterface $output) {
142			try {
143			$pages = getPages($website_url, $output);
144			} catch (Exception $exception) {
145			$output->writeln($exception->getMessage());
146			exit;
			0 ignored issues – show Best Practice introduced 2018-01-30 08:13 UTC by Report Bug Copy Issue Report Using `exit` here is not recommended. In general, usage of exit should be done with care and only when running in a scripting context like a CLI script. Loading history...
147			}
148			runProcesses($pages, 'check', $output);
149
150			saveCheckReport(parse_url($website_url, PHP_URL_HOST), $output);
151			});
152
153			$app->command('links website_url', function ($website_url, OutputInterface $output) {
154			try {
155			$pages = getPages($website_url, $output);
156			} catch (Exception $exception) {
157			$output->writeln($exception->getMessage());
158			exit;
			0 ignored issues – show Best Practice introduced 2018-01-30 08:13 UTC by Report Bug Copy Issue Report Using `exit` here is not recommended. In general, usage of exit should be done with care and only when running in a scripting context like a CLI script. Loading history...
159			};
160
161			$pdo = new PDO(DSN);
162			$pdo->query('DROP TABLE IF EXISTS links');
163			$pdo->query('CREATE TABLE links (page TEXT NOT NULL, link TEXT NOT NULL, url TEXT NOT NULL)');
164			$pdo->query('CREATE UNIQUE INDEX links_url_uindex ON links (url);');
165			unset($pdo);
166
167			runProcesses($pages, 'parse-links', $output);
168
169			$domain = parse_url($website_url, PHP_URL_HOST);
170			saveCheckReport($domain, $output);
171
172			$pdo = new PDO(DSN);
173			$pages = $pdo->query('SELECT url FROM links')->fetchAll(PDO::FETCH_COLUMN);
174			unset($pdo);
175
176			$output->writeln(' Checking founded links...');
177
178			runProcesses($pages, 'check', $output);
179
180			saveLinksReport($domain, $output);
181			});
182
183			$app->command('metadata website_url', function ($website_url, OutputInterface $output) {
184			/**
185			* @param array $tree
186			* @param callable $function
187			* @param int $level
188			* @param string $path
189			*/
190			function walker($tree, $function, $level = 0, $path = '')
191			{
192			foreach ($tree as $branchName => $branch) {
193			if (isset($branch['_self'])) {
194			$function($branch['_self'], $level, $path);
195			unset($branch['_self']);
196			}
197			if (count($branch) > 0)
198			walker($branch, $function, $level + 1, "$path/$branchName");
199			}
200			}
201
202			$output->writeln(" Getting sitemap of $website_url ...");
203			$sitemap = simplexml_load_file($website_url . '/sitemap.xml');
204
205			$resultFile = __DIR__ . '/runtime/' . parse_url($website_url, PHP_URL_HOST) . '-metadata.csv';
206
207			$paths = [];
208			foreach ($sitemap as $url) {
209			$paths[] = parse_url($url->loc, PHP_URL_PATH);
210			}
211			natsort($paths);
212
213			$tree = [];
214			foreach ($paths as $path) {
215			$levels = explode('/', $path);
216			$temp = &$tree;
217			foreach ($levels as $key => $level) {
218			// в условии неочевидное преобразование для анализа ссылки на главную страницу
219			if (!empty($level) \|\| (empty(array_filter($levels)) && $level = '/')) {
220			if (!isset($temp[$level]))
221			$temp[$level] = [];
222
223			if ($key == (count($levels) - 1))
224			$temp[$level]['_self'] = ['path' => $path];
225
226			$temp = &$temp[$level];
227			}
228			}
229			}
230			unset($temp);
231
232			file_put_contents($resultFile, 'URL, Title, Keywords, Description, "Build Time: ' . date('r') . '"' . PHP_EOL);
233			$previous = '';
234			$progress = new \cli\progress\Bar(' Getting meta data', count($paths), 1000);
235			walker($tree, function (&$self, $level, $path) use ($website_url, &$previous, $resultFile, &$progress) {
236			$data = [];
237			$page = file_get_contents($website_url . $self['path']);
238			preg_match('~<title>(.*?)</title>~', $page, $temp);
239			$data[] = $temp[1] ?? '';
240			preg_match('~<meta name="keywords" content="(.*?)">~', $page, $temp);
241			$data[] = $temp[1] ?? '';
242			preg_match('~<meta name="description" content="(.*?)">~', $page, $temp);
243			$data[] = $temp[1] ?? '';
244
245			if ($data == $previous) {
246			foreach ($data as &$item)
247			$item = '--//--';
248			} else
249			$previous = $data;
250
251			$row = [];
252			$row[] = $website_url . $self['path'];
253			$row = array_merge($row, $data);
254
255			foreach ($row as &$item)
256			$item = '"' . $item . '"';
257
258			$line = implode(',', $row) . PHP_EOL;
259			file_put_contents($resultFile, $line, FILE_APPEND);
260			$progress->tick();
261			});
262
263			$progress->finish();
264			});
265
266			/** @noinspection PhpUnhandledExceptionInspection */
267			$app->run();

cimmwolf / sitemap-parser

Push — master ( 9e19cf...b74b72 )

runProcesses() B

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like