ItemMultiResponseHandler::logToCLI() - Code Metrics - Inspection of ":octocat:" - chillerlan/php-curl - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( bc3bf5...3f770a )

by smiley

created 2016-02-22 23:45 UTC

ItemMultiResponseHandler::logToCLI() A

↳ Parent: Project

Complexity

Conditions	1
Paths	1

Size

Total Lines	3
Code Lines	2

Duplication

Lines	0
Ratio	0 %

Importance

Changes	2
Bugs	0	Features	0

Metric	Value
c	2
b	0
f	0
dl	0
loc	3
rs	10
cc	1
eloc	2
nc	1
nop	1

<?php
/**
 *
 * @filesource   ItemMultiResponseHandler.php
 * @created      16.02.2016
 * @package      Example\GW2API
 * @author       Smiley <[email protected]>
 * @copyright    2016 Smiley
 * @license      MIT
 */

namespace Example\GW2API;

use chillerlan\Database\Drivers\MySQLi\MySQLiDriver;
use chillerlan\Database\Traits\DatabaseTrait;
use chillerlan\Database\DBOptions;
use chillerlan\Database\Drivers\PDO\PDOMySQLDriver;
use chillerlan\TinyCurl\MultiRequest;
use chillerlan\TinyCurl\MultiRequestOptions;
use chillerlan\TinyCurl\Request;
use chillerlan\TinyCurl\Response\MultiResponseHandlerInterface;
use chillerlan\TinyCurl\Response\ResponseInterface;
use chillerlan\TinyCurl\URL;
use Dotenv\Dotenv;
use Exception;

/**
 * Class ItemMultiResponseHandler
 */
class ItemMultiResponseHandler implements MultiResponseHandlerInterface{
	use DatabaseTrait;

	/**
	 * class options
	 * play around with chunksize and concurrent requests to get best performance results
	 */
	const CONCURRENT    = 10;
	const CHUNK_SIZE    = 100;
	const API_LANGUAGES = ['de', 'en', 'es', 'fr', 'zh'];
	const CACERT        = __DIR__.'/../../tests/test-cacert.pem';
	const TEMP_TABLE    = 'gw2_items_temp';
	const DBDRIVER      = MySQLiDriver::class; // MySQLiDriver::class
	const API_BASE      = 'https://api.guildwars2.com/v2/items';
	const CONFIGDIR     = __DIR__.'/../../config';

	/**
	 * @var \chillerlan\TinyCurl\MultiRequest
	 */
	protected $multiRequest;

	/**
	 * @var \chillerlan\Database\Drivers\DBDriverInterface
	 */
	protected $DBDriverInterface;

	/**
	 * @var array
	 */
	protected $urls = [];

	/**
	 * @var float
	 */
	protected $starttime;

	/**
	 * @var int
	 */
	protected $callback = 0;

	/**
	 * MultiResponseHandlerTest constructor.
	 *
	 * @param \chillerlan\TinyCurl\MultiRequest $multiRequest
	 */
	public function __construct(MultiRequest $multiRequest = null){
		$this->multiRequest = $multiRequest;

		(new Dotenv(self::CONFIGDIR))->load();

		$dbOptions = new DBOptions([
			'host'     => getenv('DB_MYSQLI_HOST'),
			'port'     => getenv('DB_MYSQLI_PORT'),
			'database' => getenv('DB_MYSQLI_DATABASE'),
			'username' => getenv('DB_MYSQLI_USERNAME'),
			'password' => getenv('DB_MYSQLI_PASSWORD'),
		]);

		$this->DBDriverInterface = $this->dbconnect(self::DBDRIVER, $dbOptions);
	}

	/**
	 * start the mayhem
	 */
	public function init(){
		$this->createTempTable();
		$this->getURLs();

		$this->starttime = microtime(true);

		$options = new MultiRequestOptions;
		$options->ca_info     = self::CACERT;
		$options->base_url    = self::API_BASE.'?';
		$options->window_size = self::CONCURRENT;

		$request = new MultiRequest($options);
		// solving the hen-egg problem, feed the hen with the egg!
		$request->setHandler($this);

		$this->logToCLI('mayhem started');
		$this->callback = 0;
		$request->fetch($this->urls);
		$this->logToCLI('MultiRequest::fetch() finished');

#		var_dump($this->mySQLiDriver->raw('select * from '.self::TEMP_TABLE));
	}

	/**
	 * Schrödingers cat state handler.
	 *
	 * This method will be called within a loop in MultiRequest::processStack().
	 * You can either build your class around this MultiResponseHandlerInterface to process
	 * the response during runtime or return the response data to the running
	 * MultiRequest instance via addResponse() and receive the data by calling getResponseData().
	 *
	 * This method may return void or an URL object as a replacement for a failed request,
	 * which then will be re-added to the running queue.
	 *
	 * However, the return value will not be checked, so make sure you return valid URLs. ;)
	 *
	 * @param \chillerlan\TinyCurl\Response\ResponseInterface $response
	 *
	 * @return bool|\chillerlan\TinyCurl\URL
	 * @internal
	 */
	public function handleResponse(ResponseInterface $response){
		$info = $response->info;
		$this->callback++;

		// get the current request params
		parse_str(parse_url($info->url, PHP_URL_QUERY), $params);

		// there be dragons.
		if(in_array($info->http_code, [200, 206], true)){
			$lang = $response->headers->{'content-language'} ?: $params['lang'];

			// discard the response when it's impossible to determine the language
			if(!in_array($lang, self::API_LANGUAGES)){
				$this->logToCLI('URL discarded. ('.$info->url.')');
				return false;
			}

			$sql = 'UPDATE '.self::TEMP_TABLE.' SET `'.$lang.'` = ? WHERE `id` = ?';
			$values = [];

			foreach($response->json as $item){
#				$this->logToCLI(str_pad($item->id, 5).' - '.$item->name);
				// just dumping the raw JSON for each item here because i'm lazy (or to process the itemdata later)
				$values[] = [json_encode($item), $item->id];
			}

			// insert the data as soon as we receive it
			// this will result in a couple more database writes but won't block the responses much
			if($this->DBDriverInterface->multi($sql, $values)){
				$this->logToCLI('['.str_pad($this->callback, 6, ' ',STR_PAD_RIGHT).']['.$lang.'] '.md5($response->info->url).' updated');
			}
			else{
				// retry if the insert failed for whatever reason
				$this->logToCLI('SQL insert failed, retrying URL. ('.$info->url.')');
				return new URL($info->url);
			}

			// not adding a response if everything was fine ('s ok, PhpStorm...)
			return false;
		}
		// instant retry on a 502
		// https://gitter.im/arenanet/api-cdi?at=56c3ba6ba5bdce025f69bcc8
		else if($info->http_code === 502){
			$this->logToCLI('URL readded due to a 502. ('.$info->url.')');
			return new URL($info->url);
		}
		// examine and add the failed response to retry later @todo
		else{
			$this->logToCLI('('.$info->url.')');
			return false;
		}

	}

	/**
	 * Write some info to the CLI
	 *
	 * @param $str
	 */
	protected function logToCLI($str){
		echo '['.date('c', time()).']'.sprintf('[%10ss] ', sprintf('%01.4f', microtime(true) - $this->starttime)).$str.PHP_EOL;
	}

	/**
	 * Creates a temporary table to receive the item responses on the fly
	 */
	protected function createTempTable(){

		$sql_lang = array_map(function($lang){
			return '`'.$lang.'` text COLLATE utf8mb4_bin NOT NULL, ';
		}, self::API_LANGUAGES);

		$sql = 'CREATE TEMPORARY TABLE IF NOT EXISTS `'.self::TEMP_TABLE.'` ('
		       .'`id` int(10) unsigned NOT NULL,'
		       .substr(implode(' ', $sql_lang), 0, -1)
		       .' `updated` tinyint(1) unsigned NOT NULL DEFAULT 0,'
		       .'`response_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,'
		       .'PRIMARY KEY (`id`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin';
		$this->DBDriverInterface->raw('DROP TEMPORARY TABLE IF EXISTS `'.self::TEMP_TABLE.'`');
		$this->DBDriverInterface->raw($sql);
	}

	/**
	 * @throws \chillerlan\TinyCurl\RequestException
	 */
	protected function getURLs(){
		$this->starttime = microtime(true);
		$this->logToCLI('self::getURLs() fetch');

		$response = (new Request)->fetch(new URL('https://api.guildwars2.com/v2/items'));

		if($response->info->http_code !== 200){
			throw new Exception('failed to get /v2/items');
		}

		$values = array_map(function($item){
			return [$item];
		}, $response->json);

		$this->logToCLI('self::getURLs() $response to DB start');
		$this->DBDriverInterface->multi('INSERT INTO '.self::TEMP_TABLE.' (`id`) VALUES (?)', $values);
		$this->logToCLI('self::getURLs() $response to DB finish');

		$chunks = array_chunk($response->json, self::CHUNK_SIZE);

		array_map(function($chunk){
			foreach(self::API_LANGUAGES as $lang){
				$this->urls[] = new URL(self::API_BASE.'?lang='.$lang.'&ids='.implode(',', $chunk));
			}
		}, $chunks);

		$this->logToCLI('self::getURLs() finished');
	}

}


1			<?php
2			/**
3			*
4			* @filesource ItemMultiResponseHandler.php
5			* @created 16.02.2016
6			* @package Example\GW2API
7			* @author Smiley <[email protected]>
8			* @copyright 2016 Smiley
9			* @license MIT
10			*/
11
12			namespace Example\GW2API;
13
14			use chillerlan\Database\Drivers\MySQLi\MySQLiDriver;
15			use chillerlan\Database\Traits\DatabaseTrait;
16			use chillerlan\Database\DBOptions;
17			use chillerlan\Database\Drivers\PDO\PDOMySQLDriver;
18			use chillerlan\TinyCurl\MultiRequest;
19			use chillerlan\TinyCurl\MultiRequestOptions;
20			use chillerlan\TinyCurl\Request;
21			use chillerlan\TinyCurl\Response\MultiResponseHandlerInterface;
22			use chillerlan\TinyCurl\Response\ResponseInterface;
23			use chillerlan\TinyCurl\URL;
24			use Dotenv\Dotenv;
25			use Exception;
26
27			/**
28			* Class ItemMultiResponseHandler
29			*/
30			class ItemMultiResponseHandler implements MultiResponseHandlerInterface{
31			use DatabaseTrait;
32
33			/**
34			* class options
35			* play around with chunksize and concurrent requests to get best performance results
36			*/
37			const CONCURRENT = 10;
38			const CHUNK_SIZE = 100;
39			const API_LANGUAGES = ['de', 'en', 'es', 'fr', 'zh'];
40			const CACERT = __DIR__.'/../../tests/test-cacert.pem';
41			const TEMP_TABLE = 'gw2_items_temp';
42			const DBDRIVER = MySQLiDriver::class; // MySQLiDriver::class
43			const API_BASE = 'https://api.guildwars2.com/v2/items';
44			const CONFIGDIR = __DIR__.'/../../config';
45
46			/**
47			* @var \chillerlan\TinyCurl\MultiRequest
48			*/
49			protected $multiRequest;
50
51			/**
52			* @var \chillerlan\Database\Drivers\DBDriverInterface
53			*/
54			protected $DBDriverInterface;
55
56			/**
57			* @var array
58			*/
59			protected $urls = [];
60
61			/**
62			* @var float
63			*/
64			protected $starttime;
65
66			/**
67			* @var int
68			*/
69			protected $callback = 0;
70
71			/**
72			* MultiResponseHandlerTest constructor.
73			*
74			* @param \chillerlan\TinyCurl\MultiRequest $multiRequest
75			*/
76			public function __construct(MultiRequest $multiRequest = null){
77			$this->multiRequest = $multiRequest;
78
79			(new Dotenv(self::CONFIGDIR))->load();
80
81			$dbOptions = new DBOptions([
82			'host' => getenv('DB_MYSQLI_HOST'),
83			'port' => getenv('DB_MYSQLI_PORT'),
84			'database' => getenv('DB_MYSQLI_DATABASE'),
85			'username' => getenv('DB_MYSQLI_USERNAME'),
86			'password' => getenv('DB_MYSQLI_PASSWORD'),
87			]);
88
89			$this->DBDriverInterface = $this->dbconnect(self::DBDRIVER, $dbOptions);
90			}
91
92			/**
93			* start the mayhem
94			*/
95			public function init(){
96			$this->createTempTable();
97			$this->getURLs();
98
99			$this->starttime = microtime(true);
100
101			$options = new MultiRequestOptions;
102			$options->ca_info = self::CACERT;
103			$options->base_url = self::API_BASE.'?';
104			$options->window_size = self::CONCURRENT;
105
106			$request = new MultiRequest($options);
107			// solving the hen-egg problem, feed the hen with the egg!
108			$request->setHandler($this);
109
110			$this->logToCLI('mayhem started');
111			$this->callback = 0;
112			$request->fetch($this->urls);
113			$this->logToCLI('MultiRequest::fetch() finished');
114
115			# var_dump($this->mySQLiDriver->raw('select * from '.self::TEMP_TABLE));
116			}
117
118			/**
119			* Schrödingers cat state handler.
120			*
121			* This method will be called within a loop in MultiRequest::processStack().
122			* You can either build your class around this MultiResponseHandlerInterface to process
123			* the response during runtime or return the response data to the running
124			* MultiRequest instance via addResponse() and receive the data by calling getResponseData().
125			*
126			* This method may return void or an URL object as a replacement for a failed request,
127			* which then will be re-added to the running queue.
128			*
129			* However, the return value will not be checked, so make sure you return valid URLs. ;)
130			*
131			* @param \chillerlan\TinyCurl\Response\ResponseInterface $response
132			*
133			* @return bool\|\chillerlan\TinyCurl\URL
134			* @internal
135			*/
136			public function handleResponse(ResponseInterface $response){
137			$info = $response->info;
138			$this->callback++;
139
140			// get the current request params
141			parse_str(parse_url($info->url, PHP_URL_QUERY), $params);
142
143			// there be dragons.
144			if(in_array($info->http_code, [200, 206], true)){
145			$lang = $response->headers->{'content-language'} ?: $params['lang'];
146
147			// discard the response when it's impossible to determine the language
148			if(!in_array($lang, self::API_LANGUAGES)){
149			$this->logToCLI('URL discarded. ('.$info->url.')');
150			return false;
151			}
152
153			$sql = 'UPDATE '.self::TEMP_TABLE.' SET `'.$lang.'` = ? WHERE `id` = ?';
154			$values = [];
155
156			foreach($response->json as $item){
157			# $this->logToCLI(str_pad($item->id, 5).' - '.$item->name);
158			// just dumping the raw JSON for each item here because i'm lazy (or to process the itemdata later)
159			$values[] = [json_encode($item), $item->id];
160			}
161
162			// insert the data as soon as we receive it
163			// this will result in a couple more database writes but won't block the responses much
164			if($this->DBDriverInterface->multi($sql, $values)){
165			$this->logToCLI('['.str_pad($this->callback, 6, ' ',STR_PAD_RIGHT).']['.$lang.'] '.md5($response->info->url).' updated');
166			}
167			else{
168			// retry if the insert failed for whatever reason
169			$this->logToCLI('SQL insert failed, retrying URL. ('.$info->url.')');
170			return new URL($info->url);
171			}
172
173			// not adding a response if everything was fine ('s ok, PhpStorm...)
174			return false;
175			}
176			// instant retry on a 502
177			// https://gitter.im/arenanet/api-cdi?at=56c3ba6ba5bdce025f69bcc8
178			else if($info->http_code === 502){
179			$this->logToCLI('URL readded due to a 502. ('.$info->url.')');
180			return new URL($info->url);
181			}
182			// examine and add the failed response to retry later @todo
183			else{
184			$this->logToCLI('('.$info->url.')');
185			return false;
186			}
187
188			}
189
190			/**
191			* Write some info to the CLI
192			*
193			* @param $str
194			*/
195			protected function logToCLI($str){
196			echo '['.date('c', time()).']'.sprintf('[%10ss] ', sprintf('%01.4f', microtime(true) - $this->starttime)).$str.PHP_EOL;
197			}
198
199			/**
200			* Creates a temporary table to receive the item responses on the fly
201			*/
202			protected function createTempTable(){
203
204			$sql_lang = array_map(function($lang){
205			return '`'.$lang.'` text COLLATE utf8mb4_bin NOT NULL, ';
206			}, self::API_LANGUAGES);
207
208			$sql = 'CREATE TEMPORARY TABLE IF NOT EXISTS `'.self::TEMP_TABLE.'` ('
209			.'`id` int(10) unsigned NOT NULL,'
210			.substr(implode(' ', $sql_lang), 0, -1)
211			.' `updated` tinyint(1) unsigned NOT NULL DEFAULT 0,'
212			.'`response_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,'
213			.'PRIMARY KEY (`id`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin';
214			$this->DBDriverInterface->raw('DROP TEMPORARY TABLE IF EXISTS `'.self::TEMP_TABLE.'`');
215			$this->DBDriverInterface->raw($sql);
216			}
217
218			/**
219			* @throws \chillerlan\TinyCurl\RequestException
220			*/
221			protected function getURLs(){
222			$this->starttime = microtime(true);
223			$this->logToCLI('self::getURLs() fetch');
224
225			$response = (new Request)->fetch(new URL('https://api.guildwars2.com/v2/items'));
226
227			if($response->info->http_code !== 200){
228			throw new Exception('failed to get /v2/items');
229			}
230
231			$values = array_map(function($item){
232			return [$item];
233			}, $response->json);
234
235			$this->logToCLI('self::getURLs() $response to DB start');
236			$this->DBDriverInterface->multi('INSERT INTO '.self::TEMP_TABLE.' (`id`) VALUES (?)', $values);
237			$this->logToCLI('self::getURLs() $response to DB finish');
238
239			$chunks = array_chunk($response->json, self::CHUNK_SIZE);
240
241			array_map(function($chunk){
242			foreach(self::API_LANGUAGES as $lang){
243			$this->urls[] = new URL(self::API_BASE.'?lang='.$lang.'&ids='.implode(',', $chunk));
244			}
245			}, $chunks);
246
247			$this->logToCLI('self::getURLs() finished');
248			}
249
250			}
251

chillerlan / php-curl

Push — master ( bc3bf5...3f770a )

ItemMultiResponseHandler::logToCLI() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like