CompleteProcess::getNewRow2Complete() - Code Metrics - Inspection of "Improve CompleteProcess : inject article title + s..." - Dispositif/Wikibot - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — master ( 99b995...983f4d )

by Dispositif

created 2020-02-18 18:19 UTC

CompleteProcess::getNewRow2Complete() A

↳ Parent: CompleteProcess

Complexity

Conditions	3
Paths	2

Size

Total Lines	9
Code Lines	5

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	3
eloc	5
nc	2
nop	0
dl	0
loc	9
rs	10
c	0
b	0
f	0

<?php
/**
 * This file is part of dispositif/wikibot application
 * 2019 © Philippe M. <[email protected]>
 * For the full copyright and MIT license information, please view the LICENSE file.
 */

declare(strict_types=1);

namespace App\Application;

use App\Domain\Models\Wiki\OuvrageTemplate;
use App\Domain\OuvrageComplete;
use App\Domain\OuvrageFactory;
use App\Domain\OuvrageOptimize;
use App\Domain\Publisher\Wikidata2Ouvrage;
use App\Domain\Utils\TemplateParser;
use Normalizer;
use Throwable;

/**
 * Class CompleteProcess
 */
class CompleteProcess
{
    /**
     * @var bool
     */
    public $verbose = false;

    /**
     * @var QueueInterface
     */
    private $queueAdapter;
    /**
     * @var string
     */
    private $raw = '';
    private $page; // article title

    private $log = [];
    private $notCosmetic = false;
    private $major = false;
    /**
     * @var OuvrageTemplate
     */
    private $ouvrage;


    public function __construct(QueueInterface $queueAdapter, ?bool $verbose = false)
    {
        $this->queueAdapter = $queueAdapter;
        $this->verbose = (bool)$verbose;
    }

    public function run(?int $limit = 10000)
    {
        $memory = new Memory();
        while ($limit > 0) {
            $limit--;
            sleep(1);
            $row = $this->getNewRow2Complete();
            $this->raw = $row['raw'];
            $this->page = $row['page'];

            echo sprintf(
                "-------------------------------\n%s [%s]\n%s\n%s\n",
                date("Y-m-d H:i:s"),
                Bot::getGitVersion() ?? '',
                $this->page,
                $this->raw
            );
            if ($this->verbose) {
                $memory->echoMemory(true);
            }

            // initialise variables
            $this->log = [];
            $this->ouvrage = null;
            $this->notCosmetic = false;
            $this->major = false;


            try {
                $parse = TemplateParser::parseAllTemplateByName('ouvrage', $this->raw);
                $origin = $parse['ouvrage'][0]['model'] ?? null;
            } catch (Throwable $e) {
                echo sprintf("*** ERREUR impossible de transformer en modèle %s \n", $this->raw);
                continue;
            }

            if (!$origin instanceof OuvrageTemplate) {
                echo sprintf("*** ERREUR impossible de transformer en modèle %s \n", $this->raw);
                continue;
            }

            // Final optimizing (with online predictions)
            $optimizer = new OuvrageOptimize($origin, $this->page);
            $optimizer->doTasks();
            $this->ouvrage = $optimizer->getOuvrage();
            $this->log = array_merge($this->log, $optimizer->getLog());
            $this->notCosmetic = ($optimizer->notCosmetic || $this->notCosmetic);

            /**
             * RECHERCHE ONLINE
             */
            $isbn = $origin->getParam('isbn') ?? null; // avant mise en forme EAN>ISBN
            $isbn10 = $origin->getParam('isbn2') ?? $origin->getParam('isbn10') ?? null;
            if (!empty($isbn)
                && empty($origin->getParam('isbn invalide'))
                && empty($origin->getParam('isbn erroné'))
            ) {
                $this->onlineIsbnSearch($isbn, $isbn10);
            }

            $this->sendCompleted();
            unset($optimizer);
            unset($parse);
            unset($origin);
        } // END WHILE

        return true;
    }

    /**
     * Get array (title+raw strings) to complete from AMQP queue, SQL Select or file reading.
     *
     * @return string|null
     * @throws \Exception
     */
    private function getNewRow2Complete(): ?array
    {
        $row = $this->queueAdapter->getNewRaw();
        if (empty($row) || empty($row['raw'])) {
            echo "STOP: no more queue to process \n";
            throw new \Exception('no more queue to process');
        }

        return $row;

    }

    private function onlineIsbnSearch(string $isbn, ?string $isbn10 = null)
    {
        online:
        if ($this->verbose) {
            echo "sleep 10...\n";
        }
        sleep(10);

        try {
            if ($this->verbose) {
                dump('BIBLIO NAT FRANCE...');
            }
            // BnF sait pas trouver un vieux livre (10) d'après ISBN-13... FACEPALM !
            if ($isbn10) {
                $bnfOuvrage = OuvrageFactory::BnfFromIsbn($isbn10);
                sleep(2);
            }
            if (!$isbn10 || empty($bnfOuvrage) || empty($bnfOuvrage->getParam('titre'))) {
                $bnfOuvrage = OuvrageFactory::BnfFromIsbn($isbn);
            }
            if (isset($bnfOuvrage) and $bnfOuvrage instanceof OuvrageTemplate) {
                $this->completeOuvrage($bnfOuvrage);

                // Wikidata requests from $infos (ISBN/ISNI)
                if (!empty($bnfOuvrage->getInfos())) {
                    if ($this->verbose) {
                        dump('WIKIDATA...');
                    }
                    $wdComplete = new Wikidata2Ouvrage(clone $bnfOuvrage, $this->page);
                    $this->completeOuvrage($wdComplete->getOuvrage());
                }
            }
        } catch (Throwable $e) {
            echo sprintf(
                "*** ERREUR BnF Isbn Search %s %s %s \n",
                $e->getMessage(),
                $e->getFile(),
                $e->getLine()
            );
        }

        if (!isset($bnfOuvrage) || !$this->skipGoogle($bnfOuvrage)) {
            try {
                if ($this->verbose) {
                    dump('GOOGLE...');
                }
                $googleOuvrage = OuvrageFactory::GoogleFromIsbn($isbn);
                $this->completeOuvrage($googleOuvrage);
            } catch (Throwable $e) {
                echo "*** ERREUR GOOGLE Isbn Search ***".$e->getMessage()."\n";
                throw $e;
            }
        }

        if (!isset($bnfOuvrage) && !isset($googleOuvrage)) {
            try {
                if ($this->verbose) {
                    dump('OpenLibrary...');
                }
                $openLibraryOuvrage = OuvrageFactory::OpenLibraryFromIsbn($isbn);
                if (!empty($openLibraryOuvrage)) {
                    $this->completeOuvrage($openLibraryOuvrage);
                }
            } catch (Throwable $e) {
                echo '**** ERREUR OpenLibrary Isbn Search';
            }
        }
    }

    //    private function onlineQuerySearch(string $query)
    //    {
    //        echo "sleep 40...";
    //        sleep(20);
    //        onlineQuerySearch:
    //
    //        try {
    //            dump('GOOGLE SEARCH...');
    //            //            $googleOuvrage = OuvrageFactory::GoogleFromIsbn($isbn);
    //            $adapter = new GoogleBooksAdapter();
    //            $data = $adapter->search('blabla');
    //            dump($data);
    //            //die;
    //            //            return $import->getOuvrage();
    //            //            $this->completeOuvrage($googleOuvrage);
    //        } catch (Throwable $e) {
    //            echo "*** ERREUR GOOGLE QuerySearch *** ".$e->getMessage()."\n";
    //            echo "sleep 30min";
    //            sleep(60 * 30);
    //            echo "Wake up\n";
    //            goto onlineQuerySearch;
    //        }
    //    }

    private function completeOuvrage(OuvrageTemplate $onlineOuvrage)
    {
        if ($this->verbose) {
            dump($onlineOuvrage->serialize(true));
        }
        $optimizer = new OuvrageOptimize($onlineOuvrage, $this->page);
        $onlineOptimized = ($optimizer)->doTasks()->getOuvrage();

        $completer = new OuvrageComplete($this->ouvrage, $onlineOptimized);
        $this->ouvrage = $completer->getResult();
        if ($this->verbose) {
            dump($completer->getLog());
        }
        if ($completer->major) {
            $this->major = true;
        }
        $this->notCosmetic = ($completer->notCosmetic || $this->notCosmetic);
        $this->log = array_merge($this->log, $completer->getLog());
        unset($optimizer);
        unset($completer);
    }

    private function sendCompleted()
    {
        $isbn13 = $this->ouvrage->getParam('isbn') ?? null;

        $finalData = [
            //    'page' =>
            'raw' => $this->raw,
            'opti' => $this->serializeFinalOpti(),
            'optidate' => date("Y-m-d H:i:s"),
            'modifs' => mb_substr(implode(',', $this->log), 0, 250),
            'notcosmetic' => ($this->notCosmetic) ? 1 : 0,
            'major' => ($this->major) ? 1 : 0,
            'isbn' => substr($isbn13, 0, 20),
            'version' => Bot::getGitVersion() ?? null,
        ];
        if ($this->verbose) {
            dump($finalData);
        }
        // Json ?
        $result = $this->queueAdapter->sendCompletedData($finalData);

        dump($result); // bool
    }

    /**
     * Final serialization of the completed OuvrageTemplate.
     *
     * @return string
     */
    private function serializeFinalOpti(): string
    {
        $finalOpti = $this->ouvrage->serialize(true);
        $finalOpti = Normalizer::normalize($finalOpti);

        return $finalOpti;
    }

    private function skipGoogle($bnfOuvrage): bool
    {
        if ($bnfOuvrage instanceOf OuvrageTemplate
            && !empty($bnfOuvrage->getParam('titre'))
            && (!empty($this->ouvrage->getParam('lire en ligne'))
                || !empty($this->ouvrage->getParam('présentation en ligne')))
        ) {
            return true;
        }

        return false;
    }
}


1			<?php
2			/**
3			* This file is part of dispositif/wikibot application
4			* 2019 © Philippe M. <[email protected]>
5			* For the full copyright and MIT license information, please view the LICENSE file.
6			*/
7
8			declare(strict_types=1);
9
10			namespace App\Application;
11
12			use App\Domain\Models\Wiki\OuvrageTemplate;
13			use App\Domain\OuvrageComplete;
14			use App\Domain\OuvrageFactory;
15			use App\Domain\OuvrageOptimize;
16			use App\Domain\Publisher\Wikidata2Ouvrage;
17			use App\Domain\Utils\TemplateParser;
18			use Normalizer;
19			use Throwable;
20
21			/**
22			* Class CompleteProcess
23			*/
24			class CompleteProcess
25			{
26			/**
27			* @var bool
28			*/
29			public $verbose = false;
30
31			/**
32			* @var QueueInterface
33			*/
34			private $queueAdapter;
35			/**
36			* @var string
37			*/
38			private $raw = '';
39			private $page; // article title
40
41			private $log = [];
42			private $notCosmetic = false;
43			private $major = false;
44			/**
45			* @var OuvrageTemplate
46			*/
47			private $ouvrage;
48
49
50			public function __construct(QueueInterface $queueAdapter, ?bool $verbose = false)
51			{
52			$this->queueAdapter = $queueAdapter;
53			$this->verbose = (bool)$verbose;
54			}
55
56			public function run(?int $limit = 10000)
57			{
58			$memory = new Memory();
59			while ($limit > 0) {
60			$limit--;
61			sleep(1);
62			$row = $this->getNewRow2Complete();
63			$this->raw = $row['raw'];
64			$this->page = $row['page'];
65
66			echo sprintf(
67			"-------------------------------\n%s [%s]\n%s\n%s\n",
68			date("Y-m-d H:i:s"),
69			Bot::getGitVersion() ?? '',
70			$this->page,
71			$this->raw
72			);
73			if ($this->verbose) {
74			$memory->echoMemory(true);
75			}
76
77			// initialise variables
78			$this->log = [];
79			$this->ouvrage = null;
80			$this->notCosmetic = false;
81			$this->major = false;
82
83
84			try {
85			$parse = TemplateParser::parseAllTemplateByName('ouvrage', $this->raw);
86			$origin = $parse['ouvrage'][0]['model'] ?? null;
87			} catch (Throwable $e) {
88			echo sprintf("*** ERREUR impossible de transformer en modèle %s \n", $this->raw);
89			continue;
90			}
91
92			if (!$origin instanceof OuvrageTemplate) {
93			echo sprintf("*** ERREUR impossible de transformer en modèle %s \n", $this->raw);
94			continue;
95			}
96
97			// Final optimizing (with online predictions)
98			$optimizer = new OuvrageOptimize($origin, $this->page);
99			$optimizer->doTasks();
100			$this->ouvrage = $optimizer->getOuvrage();
101			$this->log = array_merge($this->log, $optimizer->getLog());
102			$this->notCosmetic = ($optimizer->notCosmetic \|\| $this->notCosmetic);
103
104			/**
105			* RECHERCHE ONLINE
106			*/
107			$isbn = $origin->getParam('isbn') ?? null; // avant mise en forme EAN>ISBN
108			$isbn10 = $origin->getParam('isbn2') ?? $origin->getParam('isbn10') ?? null;
109			if (!empty($isbn)
110			&& empty($origin->getParam('isbn invalide'))
111			&& empty($origin->getParam('isbn erroné'))
112			) {
113			$this->onlineIsbnSearch($isbn, $isbn10);
114			}
115
116			$this->sendCompleted();
117			unset($optimizer);
118			unset($parse);
119			unset($origin);
120			} // END WHILE
121
122			return true;
123			}
124
125			/**
126			* Get array (title+raw strings) to complete from AMQP queue, SQL Select or file reading.
127			*
128			* @return string\|null
129			* @throws \Exception
130			*/
131			private function getNewRow2Complete(): ?array
132			{
133			$row = $this->queueAdapter->getNewRaw();
134			if (empty($row) \|\| empty($row['raw'])) {
135			echo "STOP: no more queue to process \n";
136			throw new \Exception('no more queue to process');
137			}
138
139			return $row;
			1 ignored issue – show Bug Best Practice introduced 2020-02-18 18:22 UTC by Report Bug Copy Issue Report The expression `return $row` returns the type `array` which is incompatible with the documented return type `null\|string`. Loading history...
140			}
141
142			private function onlineIsbnSearch(string $isbn, ?string $isbn10 = null)
143			{
144			online:
145			if ($this->verbose) {
146			echo "sleep 10...\n";
147			}
148			sleep(10);
149
150			try {
151			if ($this->verbose) {
152			dump('BIBLIO NAT FRANCE...');
153			}
154			// BnF sait pas trouver un vieux livre (10) d'après ISBN-13... FACEPALM !
155			if ($isbn10) {
156			$bnfOuvrage = OuvrageFactory::BnfFromIsbn($isbn10);
157			sleep(2);
158			}
159			if (!$isbn10 \|\| empty($bnfOuvrage) \|\| empty($bnfOuvrage->getParam('titre'))) {
160			$bnfOuvrage = OuvrageFactory::BnfFromIsbn($isbn);
161			}
162			if (isset($bnfOuvrage) and $bnfOuvrage instanceof OuvrageTemplate) {
163			$this->completeOuvrage($bnfOuvrage);
164
165			// Wikidata requests from $infos (ISBN/ISNI)
166			if (!empty($bnfOuvrage->getInfos())) {
167			if ($this->verbose) {
168			dump('WIKIDATA...');
169			}
170			$wdComplete = new Wikidata2Ouvrage(clone $bnfOuvrage, $this->page);
171			$this->completeOuvrage($wdComplete->getOuvrage());
172			}
173			}
174			} catch (Throwable $e) {
175			echo sprintf(
176			"*** ERREUR BnF Isbn Search %s %s %s \n",
177			$e->getMessage(),
178			$e->getFile(),
179			$e->getLine()
180			);
181			}
182
183			if (!isset($bnfOuvrage) \|\| !$this->skipGoogle($bnfOuvrage)) {
184			try {
185			if ($this->verbose) {
186			dump('GOOGLE...');
187			}
188			$googleOuvrage = OuvrageFactory::GoogleFromIsbn($isbn);
189			$this->completeOuvrage($googleOuvrage);
190			} catch (Throwable $e) {
191			echo "* ERREUR GOOGLE Isbn Search *".$e->getMessage()."\n";
192			throw $e;
193			}
194			}
195
196			if (!isset($bnfOuvrage) && !isset($googleOuvrage)) {
197			try {
198			if ($this->verbose) {
199			dump('OpenLibrary...');
200			}
201			$openLibraryOuvrage = OuvrageFactory::OpenLibraryFromIsbn($isbn);
202			if (!empty($openLibraryOuvrage)) {
203			$this->completeOuvrage($openLibraryOuvrage);
204			}
205			} catch (Throwable $e) {
206			echo '**** ERREUR OpenLibrary Isbn Search';
207			}
208			}
209			}
210
211			// private function onlineQuerySearch(string $query)
212			// {
213			// echo "sleep 40...";
214			// sleep(20);
215			// onlineQuerySearch:
216			//
217			// try {
218			// dump('GOOGLE SEARCH...');
219			// // $googleOuvrage = OuvrageFactory::GoogleFromIsbn($isbn);
220			// $adapter = new GoogleBooksAdapter();
221			// $data = $adapter->search('blabla');
222			// dump($data);
223			// //die;
224			// // return $import->getOuvrage();
225			// // $this->completeOuvrage($googleOuvrage);
226			// } catch (Throwable $e) {
227			// echo "* ERREUR GOOGLE QuerySearch * ".$e->getMessage()."\n";
228			// echo "sleep 30min";
229			// sleep(60 * 30);
230			// echo "Wake up\n";
231			// goto onlineQuerySearch;
232			// }
233			// }
234
235			private function completeOuvrage(OuvrageTemplate $onlineOuvrage)
236			{
237			if ($this->verbose) {
238			dump($onlineOuvrage->serialize(true));
239			}
240			$optimizer = new OuvrageOptimize($onlineOuvrage, $this->page);
241			$onlineOptimized = ($optimizer)->doTasks()->getOuvrage();
242
243			$completer = new OuvrageComplete($this->ouvrage, $onlineOptimized);
244			$this->ouvrage = $completer->getResult();
245			if ($this->verbose) {
246			dump($completer->getLog());
247			}
248			if ($completer->major) {
249			$this->major = true;
250			}
251			$this->notCosmetic = ($completer->notCosmetic \|\| $this->notCosmetic);
252			$this->log = array_merge($this->log, $completer->getLog());
253			unset($optimizer);
254			unset($completer);
255			}
256
257			private function sendCompleted()
258			{
259			$isbn13 = $this->ouvrage->getParam('isbn') ?? null;
260
261			$finalData = [
262			// 'page' =>
263			'raw' => $this->raw,
264			'opti' => $this->serializeFinalOpti(),
265			'optidate' => date("Y-m-d H:i:s"),
266			'modifs' => mb_substr(implode(',', $this->log), 0, 250),
267			'notcosmetic' => ($this->notCosmetic) ? 1 : 0,
268			'major' => ($this->major) ? 1 : 0,
269			'isbn' => substr($isbn13, 0, 20),
270			'version' => Bot::getGitVersion() ?? null,
271			];
272			if ($this->verbose) {
273			dump($finalData);
274			}
275			// Json ?
276			$result = $this->queueAdapter->sendCompletedData($finalData);
277
278			dump($result); // bool
279			}
280
281			/**
282			* Final serialization of the completed OuvrageTemplate.
283			*
284			* @return string
285			*/
286			private function serializeFinalOpti(): string
287			{
288			$finalOpti = $this->ouvrage->serialize(true);
289			$finalOpti = Normalizer::normalize($finalOpti);
290
291			return $finalOpti;
292			}
293
294			private function skipGoogle($bnfOuvrage): bool
295			{
296			if ($bnfOuvrage instanceOf OuvrageTemplate
297			&& !empty($bnfOuvrage->getParam('titre'))
298			&& (!empty($this->ouvrage->getParam('lire en ligne'))
299			\|\| !empty($this->ouvrage->getParam('présentation en ligne')))
300			) {
301			return true;
302			}
303
304			return false;
305			}
306			}
307

Dispositif / Wikibot

Push — master ( 99b995...983f4d )

CompleteProcess::getNewRow2Complete() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like