Create::downloadPackages() - Code Metrics - Inspection of "Support for mirror without gzip" - Webysther/packagist-mirror - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — master ( 1690b4...a30145 )

by Webysther

created 2017-10-26 10:47 UTC

Create::downloadPackages() A

↳ Parent: Create

Complexity

Conditions	3
Paths	3

Size

Total Lines	55
Code Lines	36

Duplication

Lines	7
Ratio	12.73 %

Code Coverage

Tests	0
CRAP Score	12

Importance

Changes

Metric	Value
cc	3
eloc	36
nc	3
nop	0
dl	7
loc	55
ccs	0
cts	44
cp	0
crap	12
rs	9.7692
c	0
b	0
f	0

How to fix Long Method

<?php

declare(strict_types=1);

/*
 * This file is part of the Packagist Mirror.
 *
 * For the full license information, please view the LICENSE.md
 * file that was distributed with this source code.
 */

namespace Webs\Mirror\Command;

use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use GuzzleHttp\Client;
use GuzzleHttp\Psr7\Request;
use GuzzleHttp\Pool;
use stdClass;
use Generator;
use Webs\Mirror\Circular;

/**
 * Create a mirror.
 *
 * @author Webysther Nunes <[email protected]>
 */
class Create extends Base
{
    /**
     * Console description.
     *
     * @var string
     */
    protected $description = 'Create/update packagist mirror';

    /**
     * Console params configuration.
     */
    protected function configure():void
    {
        parent::configure();
        $this->setName('create')->setDescription($this->description);
    }

    /**
     * Execution.
     *
     * @param InputInterface  $input  Input console
     * @param OutputInterface $output Output console
     *
     * @return int 0 if pass, any another is error
     */
    public function childExecute(InputInterface $input, OutputInterface $output):int
    {
        $this->client = new Client([

            'base_uri' => getenv('MAIN_MIRROR').'/',
            'headers' => ['Accept-Encoding' => 'gzip'],
            'decode_content' => false,
            'timeout' => 30,
            'connect_timeout' => 15,
        ]);

        $this->hasInit = false;

        $this->loadMirrors();

        // Download providers, with repository, is incremental
        if (!$this->downloadProviders()) {
            return 1;
        }

        // Download packages
        if (!$this->downloadPackages()) {
            return 1;
        }

        // Switch .packagist.json to packagist.json
        if (!$this->switch()) {
            return 1;
        }

        // Flush old SHA256 files
        $clean = new Clean();
        if (isset($this->packages) && count($this->packages)) {
            $clean->setChangedPackage($this->packages);
        }

        if (!$clean->flush($input, $output)) {
            return 1;
        }

        if ($this->hasInit) {
            unlink(getenv('PUBLIC_DIR').'/.init');
        }

        $this->generateHtml();

        return 0;
    }

    /**
     * Load main packages.json.
     *
     * @return bool|stdClass False or the object of packages.json
     */
    protected function loadMainPackagesInformation()
    {
        $this->output->writeln(
            'Loading providers from <info>'.getenv('MAIN_MIRROR').'</>'
        );

        $response = $this->client->get('packages.json');

        // Maybe 4xx or 5xx
        if ($response->getStatusCode() >= 400) {
            $this->output->writeln('Error download source of providers');

            return false;
        }

        $json = (string) $response->getBody();
        $providers = json_decode($this->unparseGzip($json));

        if (json_last_error() !== JSON_ERROR_NONE) {
            $this->output->writeln('<error>Invalid JSON</>');

            return false;
        }

        $providers = $this->addFullPathProviders($providers);

        if (!$this->checkPackagesWasChanged($providers)) {
            return false;
        }

        if (empty($providers->{'provider-includes'})) {
            $this->output->writeln('<error>Not found providers information</>');

            return false;
        }

        return $providers;
    }

    /**
     * Check if packages.json was changed, this reduce load over main packagist.
     *
     * @return bool True if is equal, false otherside
     */
    protected function checkPackagesWasChanged($providers):bool
    {
        $cachedir = getenv('PUBLIC_DIR').'/';
        $packages = $cachedir.'packages.json.gz';
        $dotPackages = $cachedir.'.packages.json.gz';
        $newPackages = gzencode(json_encode($providers, JSON_PRETTY_PRINT));

        // if 'p/...' folder not found
        if (!file_exists($cachedir.'p')) {
            touch($cachedir.'.init');
            mkdir($cachedir.'p', 0777, true);
        }

        if (file_exists($cachedir.'.init')) {
            $this->hasInit = true;

        }

        // No provider changed? Just relax...
        if (file_exists($packages) && !$this->hasInit) {
            if (md5(file_get_contents($packages)) == md5($newPackages)) {
                $this->output->writeln('<info>Up-to-date</>');

                return false;
            }
        }

        if (!file_exists($cachedir)) {
            mkdir($cachedir, 0777, true);
        }

        if (false === file_put_contents($dotPackages, $newPackages)) {
            $this->output->writeln('<error>.packages.json not found</>');

            return false;
        }

        $this->createLink($dotPackages);

        return true;
    }

    /**
     * Switch current packagist.json to space and .packagist to packagist.json.
     *
     * @return bool True if work, false otherside
     */
    protected function switch()
    {
        $cachedir = getenv('PUBLIC_DIR').'/';
        $packages = $cachedir.'packages.json.gz';
        $dotPackages = $cachedir.'.packages.json.gz';

        if (file_exists($dotPackages)) {
            if (file_exists($packages)) {
                $this->output->writeln(
                    '<comment>Removing old packages.json</>'
                );
                unlink($packages);
            }

            $this->output->writeln(
                'Switch <info>.packages.json</> to <info>packages.json</>'
            );
            copy($dotPackages, $packages);
            $this->createLink($packages);
        }

        return true;
    }

    /**
     * Download packages.json & provider-xxx$xxx.json.
     *
     * @return bool True if work, false otherside
     */
    protected function downloadProviders():bool
    {
        if (!($providers = $this->loadMainPackagesInformation())) {
            return false;
        }

        $includes = count((array) $providers->{'provider-includes'});
        $this->progressBarStart($includes);

        $generator = $this->downloadProvideIncludes(
            $providers->{'provider-includes'}
        );

        if (!$generator->valid()) {
            $this->output->writeln('All providers up-to-date...');

            return true;
        }

        $this->errors = [];

        $this->providers = [];

        $pool = new Pool($this->client, $generator, [
            'concurrency' => getenv('MAX_CONNECTIONS'),
            'fulfilled' => function ($response, $name) {
                $json = (string) $response->getBody();
                file_put_contents($name, $json);
                $this->createLink($name);
                $this->providers[$name] = json_decode($this->unparseGzip($json));

                $this->progressBarUpdate();
            },
            'rejected' => function ($reason, $name) {
                $this->errors[$name] = $reason;

                $this->progressBarUpdate();
            },
        ]);

        // Initiate the transfers and create a promise
        $promise = $pool->promise();

        // Force the pool of requests to complete.
        $promise->wait();

        $this->progressBarUpdate();
        $this->progressBarFinish();
        $this->showErrors($this->errors);

        return true;
    }

    /**
     * Download packages.json & provider-xxx$xxx.json.
     *
     * @param stdClass $includes Providers links
     *
     * @return Generator Providers downloaded
     */
    protected function downloadProvideIncludes(stdClass $includes):Generator
    {
        $cachedir = getenv('PUBLIC_DIR').'/';

        foreach ($includes as $template => $hash) {
            $fileurl = str_replace('%hash%', $hash->sha256, $template);
            $cachename = $cachedir.$fileurl.'.gz';

            // Only if exists
            if (file_exists($cachename) && !$this->hasInit) {
                $this->progressBarUpdate();
                continue;
            }

            yield $cachename => new Request(
                'GET',
                $fileurl,
                ['curl' => [CURLMOPT_PIPELINING => 2]]

            );
        }
    }

    /**
     * Show errors formatted.
     *
     * @param array $errors Errors
     */
    protected function showErrors(array $errors):void
    {
        if (!count($errors) || $this->output->getVerbosity() < OutputInterface::VERBOSITY_VERBOSE) {
            return;
        }

        foreach ($errors as $name => $reason) {
            $shortname = $this->shortname($name);
            $error = $reason->getCode();
            $host = $reason->getRequest()->getUri()->getHost();

            $this->output->write(
                "<comment>$shortname</> failed from <info>$host</> with HTTP error"
            );

            if (!$error) {
                $this->output->writeln(
                    ':'.PHP_EOL.'<error>'.$reason->getMessage().'</>'
                );
                continue;
            }

            $this->output->writeln(" <error>$error</>");
        }

        $this->output->writeln('');
    }

    /**
     * Disable mirror when due lots of errors.
     */
    protected function disableDueErrors(array $errors)
    {
        if (!count($errors)) {
            return;
        }

        $counter = [];

        foreach ($errors as $reason) {
            $uri = $reason->getRequest()->getUri();
            $host = $uri->getScheme().'://'.$uri->getHost();

            if (!isset($counter[$host])) {
                $counter[$host] = 0;
            }

            ++$counter[$host];
        }

        $mirrors = $this->circular->toArray();
        $circular = [];

        foreach ($mirrors as $mirror) {
            if ($counter[$mirror] > 1000) {
                $this->output->writeln(
                    PHP_EOL
                    .'<error>Due to '.$counter[$mirror].' errors mirror '.$mirror.' will be disabled</>'.
                    PHP_EOL
                );
                continue;
            }

            $circular[] = $mirror;
        }

        putenv('DATA_MIRROR='.implode(',', $circular));
        $this->loadMirrors();
    }

    protected function fallback(array $files, stdClass $list, string $provider):void
    {
        $total = count($files);

        if (!$total) {
            return;
        }

        $circular = $this->circular;
        $this->circular = Circular::fromArray([getenv('MAIN_MIRROR')]);


        $shortname = $this->shortname($provider);

        $this->output->writeln(
            'Fallback packages from <info>'.$shortname.
            '</> provider to main mirror <info>'.getenv('MAIN_MIRROR').'</>'
        );

        $generator = $this->downloadPackage($list);
        $this->progressBarStart($total);
        $this->errors = [];


        $pool = new Pool($this->client, $generator, [
            'concurrency' => getenv('MAX_CONNECTIONS'),
            'fulfilled' => function ($response, $name) {

                $gzip = (string) $response->getBody();
                file_put_contents($name, $this->parseGzip($gzip));
                $this->createLink($name);
                $this->packages[] = dirname($name);

                $this->progressBarUpdate();
            },
            'rejected' => function ($reason, $name) {
                $this->errors[$name] = $reason;

                $this->progressBarUpdate();
            },
        ]);

        $promise = $pool->promise();

        // Force the pool of requests to complete.
        $promise->wait();

        $this->progressBarUpdate();
        $this->progressBarFinish();
        $this->showErrors($this->errors);
        $this->packages = array_unique($this->packages);

        $this->circular = $circular;
    }

    /**
     * Add base url of packagist.org to services on packages.json of
     * mirror don't support.
     *
     * @param stdClass $providers List of providers from packages.json
     */
    protected function addFullPathProviders(stdClass $providers):stdClass
    {
        // Add full path for services of mirror don't provide only packagist.org
        foreach (['notify', 'notify-batch', 'search'] as $key) {
            // Just in case packagist.org add full path in future
            $path = parse_url($providers->$key){'path'};
            $providers->$key = getenv('MAIN_MIRROR').$path;
        }

        return $providers;
    }

    /**
     * Download packages listed on provider-*.json on public/p dir.
     *
     * @return bool True if work, false otherside
     */
    protected function downloadPackages():bool
    {
        if (!isset($this->providers)) {
            return true;
        }

        $this->packages = [];


        $totalProviders = count($this->providers);
        $currentProvider = 0;
        foreach ($this->providers as $provider => $packages) {
            ++$currentProvider;
            $list = $packages->providers;
            $total = count((array) $list);
            $shortname = $this->shortname($provider);

            $this->output->writeln(
                '['.$currentProvider.'/'.$totalProviders.']'.
                ' Loading packages from <info>'.$shortname.'</> provider'
            );

            $generator = $this->downloadPackage($list);
            $this->progressBarStart($total);
            $this->errors = [];


            $pool = new Pool($this->client, $generator, [
                'concurrency' => getenv('MAX_CONNECTIONS') * $this->circular->count(),
                'fulfilled' => function ($response, $name) {

                    $gzip = (string) $response->getBody();
                    file_put_contents($name, $this->parseGzip($gzip));
                    $this->createLink($name);
                    $this->packages[] = dirname($name);

                    $this->progressBarUpdate();
                },
                'rejected' => function ($reason, $name) {
                    $this->errors[$name] = $reason;

                    $this->progressBarUpdate();
                },
            ]);

            // Initiate the transfers and create a promise
            $promise = $pool->promise();

            // Force the pool of requests to complete.
            $promise->wait();

            $this->progressBarUpdate();
            $this->progressBarFinish();
            $this->showErrors($this->errors);
            $this->disableDueErrors($this->errors);
            $this->fallback($this->errors, $list, $provider);
            $this->packages = array_unique($this->packages);
        }

        return true;
    }

    /**
     * Download only a package.
     *
     * @param stdClass $list Packages links
     *
     * @return Generator Providers downloaded
     */
    protected function downloadPackage(stdClass $list):Generator
    {
        $cachedir = getenv('PUBLIC_DIR').'/';
        $uri = 'p/%s$%s.json';

        foreach ($list as $name => $hash) {
            $fileurl = sprintf($uri, $name, $hash->sha256);
            $cachename = $cachedir.$fileurl.'.gz';

            // Only if exists
            if (file_exists($cachename)) {
                $this->progressBarUpdate();
                continue;
            }

            // if 'p/...' folder not found
            $subdir = dirname($cachename);
            if (!file_exists($subdir)) {
                mkdir($subdir, 0777, true);
            }

            if ($this->hasInit) {
                $fileurl = $this->circular->current().'/'.$fileurl;
                $this->circular->next();
            }

            yield $cachename => new Request(
                'GET',
                $fileurl,
                ['curl' => [CURLMOPT_PIPELINING => 2]]

            );
        }
    }

    /**
     * Generate HTML of index.html.
     */
    protected function generateHtml():void
    {
        ob_start();
        include getcwd().'/resources/index.html.php';
        file_put_contents(getenv('PUBLIC_DIR').'/index.html', ob_get_clean());
    }

    protected function loadMirrors()
    {
        $this->circular = Circular::fromArray($this->getMirrors());

    }

    protected function getMirrors():array
    {
        $mirrors = explode(',', getenv('DATA_MIRROR'));
        $mirrors[] = getenv('MAIN_MIRROR');

        return $mirrors;
    }

    /**
     * Create a simbolic link.
     *
     * @param string $path Path to file
     */
    protected function createLink(string $target):void
    {
        // From .json.gz to .json
        $link = substr($target, 0, -3);
        if (!file_exists($link)) {
            symlink(basename($target), substr($target, 0, -3));
        }
    }
}


1		<?php
2
3		declare(strict_types=1);
4
5		/*
6		* This file is part of the Packagist Mirror.
7		*
8		* For the full license information, please view the LICENSE.md
9		* file that was distributed with this source code.
10		*/
11
12		namespace Webs\Mirror\Command;
13
14		use Symfony\Component\Console\Input\InputInterface;
15		use Symfony\Component\Console\Output\OutputInterface;
16		use GuzzleHttp\Client;
17		use GuzzleHttp\Psr7\Request;
18		use GuzzleHttp\Pool;
19		use stdClass;
20		use Generator;
21		use Webs\Mirror\Circular;
22
23		/**
24		* Create a mirror.
25		*
26		* @author Webysther Nunes <[email protected]>
27		*/
28		class Create extends Base
29		{
30		/**
31		* Console description.
32		*
33		* @var string
34		*/
35		protected $description = 'Create/update packagist mirror';
36
37		/**
38		* Console params configuration.
39		*/
40		protected function configure():void
41		{
42		parent::configure();
43		$this->setName('create')->setDescription($this->description);
44		}
45
46		/**
47		* Execution.
48		*
49		* @param InputInterface $input Input console
50		* @param OutputInterface $output Output console
51		*
52		* @return int 0 if pass, any another is error
53		*/
54		public function childExecute(InputInterface $input, OutputInterface $output):int
55		{
56		$this->client = new Client([
		0 ignored issues – show Bug Best Practice introduced 2017-10-24 14:54 UTC by Report Bug Copy Issue Report The property `client` does not exist. Although not strictly required by PHP, it is generally a best practice to declare properties explicitly. Loading history...
57		'base_uri' => getenv('MAIN_MIRROR').'/',
58		'headers' => ['Accept-Encoding' => 'gzip'],
59		'decode_content' => false,
60		'timeout' => 30,
61		'connect_timeout' => 15,
62		]);
63
64		$this->hasInit = false;
		0 ignored issues – show Bug Best Practice introduced 2017-10-24 14:54 UTC by Report Bug Copy Issue Report The property `hasInit` does not exist. Although not strictly required by PHP, it is generally a best practice to declare properties explicitly. Loading history...
65		$this->loadMirrors();
66
67		// Download providers, with repository, is incremental
68		if (!$this->downloadProviders()) {
69		return 1;
70		}
71
72		// Download packages
73		if (!$this->downloadPackages()) {
74		return 1;
75		}
76
77		// Switch .packagist.json to packagist.json
78		if (!$this->switch()) {
79		return 1;
80		}
81
82		// Flush old SHA256 files
83		$clean = new Clean();
84		if (isset($this->packages) && count($this->packages)) {
85		$clean->setChangedPackage($this->packages);
86		}
87
88		if (!$clean->flush($input, $output)) {
89		return 1;
90		}
91
92		if ($this->hasInit) {
93		unlink(getenv('PUBLIC_DIR').'/.init');
94		}
95
96		$this->generateHtml();
97
98		return 0;
99		}
100
101		/**
102		* Load main packages.json.
103		*
104		* @return bool\|stdClass False or the object of packages.json
105		*/
106		protected function loadMainPackagesInformation()
107		{
108		$this->output->writeln(
109		'Loading providers from <info>'.getenv('MAIN_MIRROR').'</>'
110		);
111
112		$response = $this->client->get('packages.json');
113
114		// Maybe 4xx or 5xx
115		if ($response->getStatusCode() >= 400) {
116		$this->output->writeln('Error download source of providers');
117
118		return false;
119		}
120
121		$json = (string) $response->getBody();
122		$providers = json_decode($this->unparseGzip($json));
123
124		if (json_last_error() !== JSON_ERROR_NONE) {
125		$this->output->writeln('<error>Invalid JSON</>');
126
127		return false;
128		}
129
130		$providers = $this->addFullPathProviders($providers);
131
132		if (!$this->checkPackagesWasChanged($providers)) {
133		return false;
134		}
135
136		if (empty($providers->{'provider-includes'})) {
137		$this->output->writeln('<error>Not found providers information</>');
138
139		return false;
140		}
141
142		return $providers;
143		}
144
145		/**
146		* Check if packages.json was changed, this reduce load over main packagist.
147		*
148		* @return bool True if is equal, false otherside
149		*/
150		protected function checkPackagesWasChanged($providers):bool
151		{
152		$cachedir = getenv('PUBLIC_DIR').'/';
153		$packages = $cachedir.'packages.json.gz';
154		$dotPackages = $cachedir.'.packages.json.gz';
155		$newPackages = gzencode(json_encode($providers, JSON_PRETTY_PRINT));
156
157		// if 'p/...' folder not found
158		if (!file_exists($cachedir.'p')) {
159		touch($cachedir.'.init');
160		mkdir($cachedir.'p', 0777, true);
161		}
162
163		if (file_exists($cachedir.'.init')) {
164		$this->hasInit = true;
		0 ignored issues – show Bug Best Practice introduced 2017-10-24 14:54 UTC by Report Bug Copy Issue Report The property `hasInit` does not exist. Although not strictly required by PHP, it is generally a best practice to declare properties explicitly. Loading history...
165		}
166
167		// No provider changed? Just relax...
168		if (file_exists($packages) && !$this->hasInit) {
169		if (md5(file_get_contents($packages)) == md5($newPackages)) {
170		$this->output->writeln('<info>Up-to-date</>');
171
172		return false;
173		}
174		}
175
176		if (!file_exists($cachedir)) {
177		mkdir($cachedir, 0777, true);
178		}
179
180		if (false === file_put_contents($dotPackages, $newPackages)) {
181		$this->output->writeln('<error>.packages.json not found</>');
182
183		return false;
184		}
185
186		$this->createLink($dotPackages);
187
188		return true;
189		}
190
191		/**
192		* Switch current packagist.json to space and .packagist to packagist.json.
193		*
194		* @return bool True if work, false otherside
195		*/
196		protected function switch()
197		{
198		$cachedir = getenv('PUBLIC_DIR').'/';
199		$packages = $cachedir.'packages.json.gz';
200		$dotPackages = $cachedir.'.packages.json.gz';
201
202		if (file_exists($dotPackages)) {
203		if (file_exists($packages)) {
204		$this->output->writeln(
205		'<comment>Removing old packages.json</>'
206		);
207		unlink($packages);
208		}
209
210		$this->output->writeln(
211		'Switch <info>.packages.json</> to <info>packages.json</>'
212		);
213		copy($dotPackages, $packages);
214		$this->createLink($packages);
215		}
216
217		return true;
218		}
219
220		/**
221		* Download packages.json & provider-xxx$xxx.json.
222		*
223		* @return bool True if work, false otherside
224		*/
225		protected function downloadProviders():bool
226		{
227		if (!($providers = $this->loadMainPackagesInformation())) {
228		return false;
229		}
230
231		$includes = count((array) $providers->{'provider-includes'});
232		$this->progressBarStart($includes);
233
234		$generator = $this->downloadProvideIncludes(
235		$providers->{'provider-includes'}
236		);
237
238		if (!$generator->valid()) {
239		$this->output->writeln('All providers up-to-date...');
240
241		return true;
242		}
243
244		$this->errors = [];
		0 ignored issues – show Bug Best Practice introduced 2017-10-24 14:54 UTC by Report Bug Copy Issue Report The property `errors` does not exist. Although not strictly required by PHP, it is generally a best practice to declare properties explicitly. Loading history...
245		$this->providers = [];
		0 ignored issues – show Bug Best Practice introduced 2017-10-24 14:54 UTC by Report Bug Copy Issue Report The property `providers` does not exist. Although not strictly required by PHP, it is generally a best practice to declare properties explicitly. Loading history...
246		$pool = new Pool($this->client, $generator, [
247		'concurrency' => getenv('MAX_CONNECTIONS'),
248		'fulfilled' => function ($response, $name) {
249		$json = (string) $response->getBody();
250		file_put_contents($name, $json);
251		$this->createLink($name);
252		$this->providers[$name] = json_decode($this->unparseGzip($json));
		0 ignored issues – show Bug Best Practice introduced 2017-10-26 10:49 UTC by Report Bug Copy Issue Report The property `providers` does not exist. Although not strictly required by PHP, it is generally a best practice to declare properties explicitly. Loading history...
253		$this->progressBarUpdate();
254		},
255		'rejected' => function ($reason, $name) {
256		$this->errors[$name] = $reason;
		0 ignored issues – show Bug Best Practice introduced 2017-10-24 14:54 UTC by Report Bug Copy Issue Report The property `errors` does not exist. Although not strictly required by PHP, it is generally a best practice to declare properties explicitly. Loading history...
257		$this->progressBarUpdate();
258		},
259		]);
260
261		// Initiate the transfers and create a promise
262		$promise = $pool->promise();
263
264		// Force the pool of requests to complete.
265		$promise->wait();
266
267		$this->progressBarUpdate();
268		$this->progressBarFinish();
269		$this->showErrors($this->errors);
270
271		return true;
272		}
273
274		/**
275		* Download packages.json & provider-xxx$xxx.json.
276		*
277		* @param stdClass $includes Providers links
278		*
279		* @return Generator Providers downloaded
280		*/
281		protected function downloadProvideIncludes(stdClass $includes):Generator
282		{
283		$cachedir = getenv('PUBLIC_DIR').'/';
284
285		foreach ($includes as $template => $hash) {
286		$fileurl = str_replace('%hash%', $hash->sha256, $template);
287		$cachename = $cachedir.$fileurl.'.gz';
288
289		// Only if exists
290		if (file_exists($cachename) && !$this->hasInit) {
291		$this->progressBarUpdate();
292		continue;
293		}
294
295		yield $cachename => new Request(
296		'GET',
297		$fileurl,
298		['curl' => [CURLMOPT_PIPELINING => 2]]
		0 ignored issues – show Bug introduced 2017-10-24 14:54 UTC by Report Bug Copy Issue Report The constant `Webs\Mirror\Command\CURLMOPT_PIPELINING` was not found. Maybe you did not declare it correctly or list all dependencies? Loading history...
299		);
300		}
301		}
302
303		/**
304		* Show errors formatted.
305		*
306		* @param array $errors Errors
307		*/
308		protected function showErrors(array $errors):void
309		{
310		if (!count($errors) \|\| $this->output->getVerbosity() < OutputInterface::VERBOSITY_VERBOSE) {
311		return;
312		}
313
314		foreach ($errors as $name => $reason) {
315		$shortname = $this->shortname($name);
316		$error = $reason->getCode();
317		$host = $reason->getRequest()->getUri()->getHost();
318
319		$this->output->write(
320		"<comment>$shortname</> failed from <info>$host</> with HTTP error"
321		);
322
323		if (!$error) {
324		$this->output->writeln(
325		':'.PHP_EOL.'<error>'.$reason->getMessage().'</>'
326		);
327		continue;
328		}
329
330		$this->output->writeln(" <error>$error</>");
331		}
332
333		$this->output->writeln('');
334		}
335
336		/**
337		* Disable mirror when due lots of errors.
338		*/
339		protected function disableDueErrors(array $errors)
340		{
341		if (!count($errors)) {
342		return;
343		}
344
345		$counter = [];
346
347		foreach ($errors as $reason) {
348		$uri = $reason->getRequest()->getUri();
349		$host = $uri->getScheme().'://'.$uri->getHost();
350
351		if (!isset($counter[$host])) {
352		$counter[$host] = 0;
353		}
354
355		++$counter[$host];
356		}
357
358		$mirrors = $this->circular->toArray();
359		$circular = [];
360
361		foreach ($mirrors as $mirror) {
362		if ($counter[$mirror] > 1000) {
363		$this->output->writeln(
364		PHP_EOL
365		.'<error>Due to '.$counter[$mirror].' errors mirror '.$mirror.' will be disabled</>'.
366		PHP_EOL
367		);
368		continue;
369		}
370
371		$circular[] = $mirror;
372		}
373
374		putenv('DATA_MIRROR='.implode(',', $circular));
375		$this->loadMirrors();
376		}
377
378		protected function fallback(array $files, stdClass $list, string $provider):void
379		{
380		$total = count($files);
381
382		if (!$total) {
383		return;
384		}
385
386		$circular = $this->circular;
387		$this->circular = Circular::fromArray([getenv('MAIN_MIRROR')]);
		0 ignored issues – show Bug Best Practice introduced 2017-10-24 14:54 UTC by Report Bug Copy Issue Report The property `circular` does not exist. Although not strictly required by PHP, it is generally a best practice to declare properties explicitly. Loading history...
388
389		$shortname = $this->shortname($provider);
390
391		$this->output->writeln(
392		'Fallback packages from <info>'.$shortname.
393		'</> provider to main mirror <info>'.getenv('MAIN_MIRROR').'</>'
394		);
395
396		$generator = $this->downloadPackage($list);
397		$this->progressBarStart($total);
398		$this->errors = [];
		0 ignored issues – show Bug Best Practice introduced 2017-10-24 14:54 UTC by Report Bug Copy Issue Report The property `errors` does not exist. Although not strictly required by PHP, it is generally a best practice to declare properties explicitly. Loading history...
399
400		$pool = new Pool($this->client, $generator, [
401		'concurrency' => getenv('MAX_CONNECTIONS'),
402	View Code Duplication	'fulfilled' => function ($response, $name) {
		0 ignored issues – show Duplication introduced 2017-10-24 14:54 UTC by Report Bug Copy Issue Report This code seems to be duplicated across your project. Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation. You can also find more detailed suggestions in the “Code” section of your repository. Loading history...
403		$gzip = (string) $response->getBody();
404		file_put_contents($name, $this->parseGzip($gzip));
405		$this->createLink($name);
406		$this->packages[] = dirname($name);
		0 ignored issues – show Bug Best Practice introduced 2017-10-24 14:54 UTC by Report Bug Copy Issue Report The property `packages` does not exist. Although not strictly required by PHP, it is generally a best practice to declare properties explicitly. Loading history...
407		$this->progressBarUpdate();
408		},
409		'rejected' => function ($reason, $name) {
410		$this->errors[$name] = $reason;
		0 ignored issues – show Bug Best Practice introduced 2017-10-24 14:54 UTC by Report Bug Copy Issue Report The property `errors` does not exist. Although not strictly required by PHP, it is generally a best practice to declare properties explicitly. Loading history...
411		$this->progressBarUpdate();
412		},
413		]);
414
415		$promise = $pool->promise();
416
417		// Force the pool of requests to complete.
418		$promise->wait();
419
420		$this->progressBarUpdate();
421		$this->progressBarFinish();
422		$this->showErrors($this->errors);
423		$this->packages = array_unique($this->packages);
		0 ignored issues – show Bug Best Practice introduced 2017-10-24 14:54 UTC by Report Bug Copy Issue Report The property `packages` does not exist. Although not strictly required by PHP, it is generally a best practice to declare properties explicitly. Loading history...
424		$this->circular = $circular;
425		}
426
427		/**
428		* Add base url of packagist.org to services on packages.json of
429		* mirror don't support.
430		*
431		* @param stdClass $providers List of providers from packages.json
432		*/
433		protected function addFullPathProviders(stdClass $providers):stdClass
434		{
435		// Add full path for services of mirror don't provide only packagist.org
436		foreach (['notify', 'notify-batch', 'search'] as $key) {
437		// Just in case packagist.org add full path in future
438		$path = parse_url($providers->$key){'path'};
439		$providers->$key = getenv('MAIN_MIRROR').$path;
440		}
441
442		return $providers;
443		}
444
445		/**
446		* Download packages listed on provider-*.json on public/p dir.
447		*
448		* @return bool True if work, false otherside
449		*/
450		protected function downloadPackages():bool
451		{
452		if (!isset($this->providers)) {
453		return true;
454		}
455
456		$this->packages = [];
		0 ignored issues – show Bug Best Practice introduced 2017-10-24 14:54 UTC by Report Bug Copy Issue Report The property `packages` does not exist. Although not strictly required by PHP, it is generally a best practice to declare properties explicitly. Loading history...
457
458		$totalProviders = count($this->providers);
459		$currentProvider = 0;
460		foreach ($this->providers as $provider => $packages) {
461		++$currentProvider;
462		$list = $packages->providers;
463		$total = count((array) $list);
464		$shortname = $this->shortname($provider);
465
466		$this->output->writeln(
467		'['.$currentProvider.'/'.$totalProviders.']'.
468		' Loading packages from <info>'.$shortname.'</> provider'
469		);
470
471		$generator = $this->downloadPackage($list);
472		$this->progressBarStart($total);
473		$this->errors = [];
		0 ignored issues – show Bug Best Practice introduced 2017-10-24 14:54 UTC by Report Bug Copy Issue Report The property `errors` does not exist. Although not strictly required by PHP, it is generally a best practice to declare properties explicitly. Loading history...
474
475		$pool = new Pool($this->client, $generator, [
476		'concurrency' => getenv('MAX_CONNECTIONS') * $this->circular->count(),
477	View Code Duplication	'fulfilled' => function ($response, $name) {
		0 ignored issues – show Duplication introduced 2017-10-24 14:54 UTC by Report Bug Copy Issue Report This code seems to be duplicated across your project. Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation. You can also find more detailed suggestions in the “Code” section of your repository. Loading history...
478		$gzip = (string) $response->getBody();
479		file_put_contents($name, $this->parseGzip($gzip));
480		$this->createLink($name);
481		$this->packages[] = dirname($name);
		0 ignored issues – show Bug Best Practice introduced 2017-10-24 14:54 UTC by Report Bug Copy Issue Report The property `packages` does not exist. Although not strictly required by PHP, it is generally a best practice to declare properties explicitly. Loading history...
482		$this->progressBarUpdate();
483		},
484		'rejected' => function ($reason, $name) {
485		$this->errors[$name] = $reason;
		0 ignored issues – show Bug Best Practice introduced 2017-10-24 14:54 UTC by Report Bug Copy Issue Report The property `errors` does not exist. Although not strictly required by PHP, it is generally a best practice to declare properties explicitly. Loading history...
486		$this->progressBarUpdate();
487		},
488		]);
489
490		// Initiate the transfers and create a promise
491		$promise = $pool->promise();
492
493		// Force the pool of requests to complete.
494		$promise->wait();
495
496		$this->progressBarUpdate();
497		$this->progressBarFinish();
498		$this->showErrors($this->errors);
499		$this->disableDueErrors($this->errors);
500		$this->fallback($this->errors, $list, $provider);
501		$this->packages = array_unique($this->packages);
502		}
503
504		return true;
505		}
506
507		/**
508		* Download only a package.
509		*
510		* @param stdClass $list Packages links
511		*
512		* @return Generator Providers downloaded
513		*/
514		protected function downloadPackage(stdClass $list):Generator
515		{
516		$cachedir = getenv('PUBLIC_DIR').'/';
517		$uri = 'p/%s$%s.json';
518
519		foreach ($list as $name => $hash) {
520		$fileurl = sprintf($uri, $name, $hash->sha256);
521		$cachename = $cachedir.$fileurl.'.gz';
522
523		// Only if exists
524		if (file_exists($cachename)) {
525		$this->progressBarUpdate();
526		continue;
527		}
528
529		// if 'p/...' folder not found
530		$subdir = dirname($cachename);
531		if (!file_exists($subdir)) {
532		mkdir($subdir, 0777, true);
533		}
534
535		if ($this->hasInit) {
536		$fileurl = $this->circular->current().'/'.$fileurl;
537		$this->circular->next();
538		}
539
540		yield $cachename => new Request(
541		'GET',
542		$fileurl,
543		['curl' => [CURLMOPT_PIPELINING => 2]]
		0 ignored issues – show Bug introduced 2017-10-24 14:54 UTC by Report Bug Copy Issue Report The constant `Webs\Mirror\Command\CURLMOPT_PIPELINING` was not found. Maybe you did not declare it correctly or list all dependencies? Loading history...
544		);
545		}
546		}
547
548		/**
549		* Generate HTML of index.html.
550		*/
551		protected function generateHtml():void
552		{
553		ob_start();
554		include getcwd().'/resources/index.html.php';
555		file_put_contents(getenv('PUBLIC_DIR').'/index.html', ob_get_clean());
556		}
557
558		protected function loadMirrors()
559		{
560		$this->circular = Circular::fromArray($this->getMirrors());
		0 ignored issues – show Bug Best Practice introduced 2017-10-24 14:54 UTC by Report Bug Copy Issue Report The property `circular` does not exist. Although not strictly required by PHP, it is generally a best practice to declare properties explicitly. Loading history...
561		}
562
563		protected function getMirrors():array
564		{
565		$mirrors = explode(',', getenv('DATA_MIRROR'));
566		$mirrors[] = getenv('MAIN_MIRROR');
567
568		return $mirrors;
569		}
570
571		/**
572		* Create a simbolic link.
573		*
574		* @param string $path Path to file
575		*/
576		protected function createLink(string $target):void
577		{
578		// From .json.gz to .json
579		$link = substr($target, 0, -3);
580		if (!file_exists($link)) {
581		symlink(basename($target), substr($target, 0, -3));
582		}
583		}
584		}
585

Webysther / packagist-mirror

Push — master ( 1690b4...a30145 )

Create::downloadPackages() A

Complexity

Size

Duplication

Code Coverage

Importance

How to fix Long Method

Long Method

Duplication Side-by-Side

Filter issues like