1
|
|
|
<?php |
2
|
|
|
declare(strict_types = 1); |
3
|
|
|
|
4
|
|
|
namespace BrowscapPHP\Command; |
5
|
|
|
|
6
|
|
|
use BrowscapPHP\Browscap; |
7
|
|
|
use BrowscapPHP\Exception\InvalidArgumentException; |
8
|
|
|
use BrowscapPHP\Exception\ReaderException; |
9
|
|
|
use BrowscapPHP\Exception\UnknownBrowserException; |
10
|
|
|
use BrowscapPHP\Exception\UnknownBrowserTypeException; |
11
|
|
|
use BrowscapPHP\Exception\UnknownDeviceException; |
12
|
|
|
use BrowscapPHP\Exception\UnknownEngineException; |
13
|
|
|
use BrowscapPHP\Exception\UnknownPlatformException; |
14
|
|
|
use BrowscapPHP\Helper\Filesystem; |
15
|
|
|
use BrowscapPHP\Helper\LoggerHelper; |
16
|
|
|
use BrowscapPHP\Util\Logfile\ReaderCollection; |
17
|
|
|
use BrowscapPHP\Util\Logfile\ReaderFactory; |
18
|
|
|
use Doctrine\Common\Cache\FilesystemCache; |
19
|
|
|
use Roave\DoctrineSimpleCache\SimpleCacheAdapter; |
20
|
|
|
use Symfony\Component\Console\Command\Command; |
21
|
|
|
use Symfony\Component\Console\Input\InputArgument; |
22
|
|
|
use Symfony\Component\Console\Input\InputInterface; |
23
|
|
|
use Symfony\Component\Console\Input\InputOption; |
24
|
|
|
use Symfony\Component\Console\Output\OutputInterface; |
25
|
|
|
use Symfony\Component\Filesystem\Exception\IOException; |
26
|
|
|
use Symfony\Component\Finder\Finder; |
27
|
|
|
use Symfony\Component\Finder\SplFileInfo; |
28
|
|
|
|
29
|
|
|
/** |
30
|
|
|
* Commands to parse a log file and parse the useragents in it |
31
|
|
|
*/ |
32
|
|
|
class LogfileCommand extends Command |
33
|
|
|
{ |
34
|
|
|
/** |
35
|
|
|
* @var array |
36
|
|
|
*/ |
37
|
|
|
private $undefinedClients = []; |
38
|
|
|
|
39
|
|
|
/** |
40
|
|
|
* @var array |
41
|
|
|
*/ |
42
|
|
|
private $uas = []; |
43
|
|
|
|
44
|
|
|
/** |
45
|
|
|
* @var array |
46
|
|
|
*/ |
47
|
|
|
private $uasWithType = []; |
48
|
|
|
|
49
|
|
|
/** |
50
|
|
|
* @var int |
51
|
|
|
*/ |
52
|
|
|
private $countOk = 0; |
53
|
|
|
|
54
|
|
|
/** |
55
|
|
|
* @var int |
56
|
|
|
*/ |
57
|
|
|
private $countNok = 0; |
58
|
|
|
|
59
|
|
|
/** |
60
|
|
|
* @var int |
61
|
|
|
*/ |
62
|
|
|
private $totalCount = 0; |
63
|
|
|
|
64
|
|
|
/** |
65
|
|
|
* @var string |
66
|
|
|
*/ |
67
|
|
|
private $defaultCacheFolder; |
68
|
|
|
|
69
|
|
|
public function __construct(string $defaultCacheFolder) |
70
|
|
|
{ |
71
|
|
|
$this->defaultCacheFolder = $defaultCacheFolder; |
72
|
|
|
|
73
|
|
|
parent::__construct(); |
74
|
|
|
} |
75
|
|
|
|
76
|
1 |
|
protected function configure() : void |
77
|
|
|
{ |
78
|
|
|
$this |
79
|
1 |
|
->setName('browscap:log') |
80
|
1 |
|
->setDescription('Parses the supplied webserver log file.') |
81
|
1 |
|
->addArgument( |
82
|
1 |
|
'output', |
83
|
1 |
|
InputArgument::REQUIRED, |
84
|
1 |
|
'Path to output log file', |
85
|
1 |
|
null |
86
|
|
|
) |
87
|
1 |
|
->addOption( |
88
|
1 |
|
'log-file', |
89
|
1 |
|
'f', |
90
|
1 |
|
InputOption::VALUE_REQUIRED, |
91
|
1 |
|
'Path to a webserver log file' |
92
|
|
|
) |
93
|
1 |
|
->addOption( |
94
|
1 |
|
'log-dir', |
95
|
1 |
|
'd', |
96
|
1 |
|
InputOption::VALUE_REQUIRED, |
97
|
1 |
|
'Path to webserver log directory' |
98
|
|
|
) |
99
|
1 |
|
->addOption( |
100
|
1 |
|
'include', |
101
|
1 |
|
'i', |
102
|
1 |
|
InputOption::VALUE_IS_ARRAY | InputOption::VALUE_REQUIRED, |
103
|
1 |
|
'Include glob expressions for log files in the log directory', |
104
|
1 |
|
['*.log', '*.log*.gz', '*.log*.bz2'] |
105
|
|
|
) |
106
|
1 |
|
->addOption( |
107
|
1 |
|
'exclude', |
108
|
1 |
|
'e', |
109
|
1 |
|
InputOption::VALUE_IS_ARRAY | InputOption::VALUE_REQUIRED, |
110
|
1 |
|
'Exclude glob expressions for log files in the log directory', |
111
|
1 |
|
['*error*'] |
112
|
|
|
) |
113
|
1 |
|
->addOption( |
114
|
1 |
|
'cache', |
115
|
1 |
|
'c', |
116
|
1 |
|
InputOption::VALUE_OPTIONAL, |
117
|
1 |
|
'Where the cache files are located', |
118
|
1 |
|
$this->defaultCacheFolder |
119
|
|
|
); |
120
|
1 |
|
} |
121
|
|
|
|
122
|
|
|
protected function execute(InputInterface $input, OutputInterface $output) : void |
123
|
|
|
{ |
124
|
|
|
if (! $input->getOption('log-file') && ! $input->getOption('log-dir')) { |
125
|
|
|
throw InvalidArgumentException::oneOfCommandArguments('log-file', 'log-dir'); |
126
|
|
|
} |
127
|
|
|
|
128
|
|
|
$logger = LoggerHelper::createDefaultLogger($output); |
129
|
|
|
|
130
|
|
|
$fileCache = new FilesystemCache($input->getOption('cache')); |
131
|
|
|
$cache = new SimpleCacheAdapter($fileCache); |
132
|
|
|
|
133
|
|
|
$browscap = new Browscap($cache, $logger); |
134
|
|
|
$collection = ReaderFactory::factory(); |
135
|
|
|
$fs = new Filesystem(); |
136
|
|
|
|
137
|
|
|
/** @var \Symfony\Component\Finder\SplFileInfo $file */ |
138
|
|
|
foreach ($this->getFiles($input) as $file) { |
139
|
|
|
$this->uas = []; |
140
|
|
|
$path = $this->getPath($file); |
141
|
|
|
|
142
|
|
|
$this->countOk = 0; |
143
|
|
|
$this->countNok = 0; |
144
|
|
|
|
145
|
|
|
$logger->info('Analyzing file "' . $file->getPathname() . '"'); |
146
|
|
|
|
147
|
|
|
$lines = file($path); |
148
|
|
|
|
149
|
|
|
if (empty($lines)) { |
150
|
|
|
$logger->info('Skipping empty file "' . $file->getPathname() . '"'); |
151
|
|
|
|
152
|
|
|
continue; |
153
|
|
|
} |
154
|
|
|
|
155
|
|
|
$this->totalCount = count($lines); |
156
|
|
|
|
157
|
|
|
foreach ($lines as $line) { |
158
|
|
|
$this->handleLine( |
159
|
|
|
$output, |
160
|
|
|
$collection, |
161
|
|
|
$browscap, |
162
|
|
|
$line |
163
|
|
|
); |
164
|
|
|
} |
165
|
|
|
|
166
|
|
|
$this->outputProgress($output, '', true); |
167
|
|
|
|
168
|
|
|
arsort($this->uas, SORT_NUMERIC); |
169
|
|
|
|
170
|
|
|
try { |
171
|
|
|
$fs->dumpFile( |
172
|
|
|
$input->getArgument('output') . '/output.txt', |
173
|
|
|
implode(PHP_EOL, array_unique($this->undefinedClients)) |
174
|
|
|
); |
175
|
|
|
} catch (IOException $e) { |
|
|
|
|
176
|
|
|
// do nothing |
177
|
|
|
} |
178
|
|
|
|
179
|
|
|
try { |
180
|
|
|
$fs->dumpFile( |
181
|
|
|
$input->getArgument('output') . '/output-with-amount.txt', |
182
|
|
|
$this->createAmountContent() |
183
|
|
|
); |
184
|
|
|
} catch (IOException $e) { |
|
|
|
|
185
|
|
|
// do nothing |
186
|
|
|
} |
187
|
|
|
|
188
|
|
|
try { |
189
|
|
|
$fs->dumpFile( |
190
|
|
|
$input->getArgument('output') . '/output-with-amount-and-type.txt', |
191
|
|
|
$this->createAmountTypeContent() |
192
|
|
|
); |
193
|
|
|
} catch (IOException $e) { |
|
|
|
|
194
|
|
|
// do nothing |
195
|
|
|
} |
196
|
|
|
} |
197
|
|
|
|
198
|
|
|
$outputFile = $input->getArgument('output') . '/output.txt'; |
199
|
|
|
|
200
|
|
|
try { |
201
|
|
|
$fs->dumpFile( |
202
|
|
|
$outputFile, |
203
|
|
|
implode(PHP_EOL, array_unique($this->undefinedClients)) |
204
|
|
|
); |
205
|
|
|
} catch (IOException $e) { |
|
|
|
|
206
|
|
|
throw new \UnexpectedValueException('writing to file "' . $outputFile . '" failed', 0, $e); |
207
|
|
|
} |
208
|
|
|
|
209
|
|
|
try { |
210
|
|
|
$fs->dumpFile( |
211
|
|
|
$input->getArgument('output') . '/output-with-amount.txt', |
212
|
|
|
$this->createAmountContent() |
213
|
|
|
); |
214
|
|
|
} catch (IOException $e) { |
|
|
|
|
215
|
|
|
// do nothing |
216
|
|
|
} |
217
|
|
|
|
218
|
|
|
try { |
219
|
|
|
$fs->dumpFile( |
220
|
|
|
$input->getArgument('output') . '/output-with-amount-and-type.txt', |
221
|
|
|
$this->createAmountTypeContent() |
222
|
|
|
); |
223
|
|
|
} catch (IOException $e) { |
|
|
|
|
224
|
|
|
// do nothing |
225
|
|
|
} |
226
|
|
|
} |
227
|
|
|
|
228
|
|
|
private function createAmountContent() : string |
229
|
|
|
{ |
230
|
|
|
$counts = []; |
231
|
|
|
|
232
|
|
|
foreach ($this->uasWithType as $uas) { |
233
|
|
|
foreach ($uas as $userAgentString => $count) { |
234
|
|
|
if (isset($counts[$userAgentString])) { |
235
|
|
|
$counts[$userAgentString] += $count; |
236
|
|
|
} else { |
237
|
|
|
$counts[$userAgentString] = $count; |
238
|
|
|
} |
239
|
|
|
} |
240
|
|
|
} |
241
|
|
|
|
242
|
|
|
$content = ''; |
243
|
|
|
|
244
|
|
|
arsort($counts, SORT_NUMERIC); |
245
|
|
|
|
246
|
|
|
foreach ($counts as $agentOfLine => $count) { |
247
|
|
|
$content .= "$count\t$agentOfLine\n"; |
248
|
|
|
} |
249
|
|
|
|
250
|
|
|
return $content; |
251
|
|
|
} |
252
|
|
|
|
253
|
|
|
private function createAmountTypeContent() : string |
254
|
|
|
{ |
255
|
|
|
$content = ''; |
256
|
|
|
$types = ['B', 'T', 'P', 'D', 'N', 'U']; |
257
|
|
|
|
258
|
|
|
foreach ($types as $type) { |
259
|
|
|
if (! isset($this->uasWithType[$type])) { |
260
|
|
|
continue; |
261
|
|
|
} |
262
|
|
|
|
263
|
|
|
arsort($this->uasWithType[$type], SORT_NUMERIC); |
264
|
|
|
|
265
|
|
|
foreach ($this->uasWithType[$type] as $agentOfLine => $count) { |
266
|
|
|
$content .= "$type\t$count\t$agentOfLine\n"; |
267
|
|
|
} |
268
|
|
|
} |
269
|
|
|
|
270
|
|
|
return $content; |
271
|
|
|
} |
272
|
|
|
|
273
|
|
|
private function handleLine( |
274
|
|
|
OutputInterface $output, |
275
|
|
|
ReaderCollection $collection, |
276
|
|
|
Browscap $browscap, |
277
|
|
|
string $line |
278
|
|
|
) : void { |
279
|
|
|
$userAgentString = ''; |
280
|
|
|
|
281
|
|
|
try { |
282
|
|
|
$userAgentString = $collection->read($line); |
283
|
|
|
|
284
|
|
|
try { |
285
|
|
|
$this->getResult($browscap->getBrowser($userAgentString)); |
286
|
|
|
} catch (\Exception $e) { |
287
|
|
|
$this->undefinedClients[] = $userAgentString; |
288
|
|
|
|
289
|
|
|
throw $e; |
290
|
|
|
} |
291
|
|
|
|
292
|
|
|
$type = '.'; |
293
|
|
|
++$this->countOk; |
294
|
|
|
} catch (ReaderException $e) { |
295
|
|
|
$type = 'E'; |
296
|
|
|
++$this->countNok; |
297
|
|
|
} catch (UnknownBrowserTypeException $e) { |
298
|
|
|
$type = 'T'; |
299
|
|
|
++$this->countNok; |
300
|
|
|
} catch (UnknownBrowserException $e) { |
301
|
|
|
$type = 'B'; |
302
|
|
|
++$this->countNok; |
303
|
|
|
} catch (UnknownPlatformException $e) { |
304
|
|
|
$type = 'P'; |
305
|
|
|
++$this->countNok; |
306
|
|
|
} catch (UnknownDeviceException $e) { |
307
|
|
|
$type = 'D'; |
308
|
|
|
++$this->countNok; |
309
|
|
|
} catch (UnknownEngineException $e) { |
310
|
|
|
$type = 'N'; |
311
|
|
|
++$this->countNok; |
312
|
|
|
} catch (\Exception $e) { |
313
|
|
|
$type = 'U'; |
314
|
|
|
++$this->countNok; |
315
|
|
|
} |
316
|
|
|
|
317
|
|
|
$this->outputProgress($output, $type); |
318
|
|
|
|
319
|
|
|
// count all useragents |
320
|
|
|
if (isset($this->uas[$userAgentString])) { |
321
|
|
|
++$this->uas[$userAgentString]; |
322
|
|
|
} else { |
323
|
|
|
$this->uas[$userAgentString] = 1; |
324
|
|
|
} |
325
|
|
|
|
326
|
|
|
if ('.' !== $type && 'E' !== $type) { |
327
|
|
|
// count all undetected useragents grouped by detection error |
328
|
|
|
if (! isset($this->uasWithType[$type])) { |
329
|
|
|
$this->uasWithType[$type] = []; |
330
|
|
|
} |
331
|
|
|
|
332
|
|
|
if (isset($this->uasWithType[$type][$userAgentString])) { |
333
|
|
|
++$this->uasWithType[$type][$userAgentString]; |
334
|
|
|
} else { |
335
|
|
|
$this->uasWithType[$type][$userAgentString] = 1; |
336
|
|
|
} |
337
|
|
|
} |
338
|
|
|
} |
339
|
|
|
|
340
|
|
|
private function outputProgress(OutputInterface $output, string $result, bool $end = false) : void |
341
|
|
|
{ |
342
|
|
|
if (0 === ($this->totalCount % 70) || $end) { |
343
|
|
|
$formatString = ' %' . strlen($this->countOk) . 'd OK, %' . strlen($this->countNok) . 'd NOK, Summary %' |
344
|
|
|
. strlen($this->totalCount) . 'd'; |
345
|
|
|
|
346
|
|
|
if ($end) { |
347
|
|
|
$result = str_pad($result, 70 - ($this->totalCount % 70), ' ', STR_PAD_RIGHT); |
348
|
|
|
} |
349
|
|
|
|
350
|
|
|
$endString = sprintf($formatString, $this->countOk, $this->countNok, $this->totalCount); |
351
|
|
|
|
352
|
|
|
$output->writeln($result . $endString); |
353
|
|
|
|
354
|
|
|
return; |
355
|
|
|
} |
356
|
|
|
|
357
|
|
|
$output->write($result); |
358
|
|
|
} |
359
|
|
|
|
360
|
|
|
private function getResult(\stdClass $result) : string |
361
|
|
|
{ |
362
|
|
|
if ('Default Browser' === $result->browser) { |
363
|
|
|
throw new UnknownBrowserException('Unknown browser found'); |
364
|
|
|
} |
365
|
|
|
|
366
|
|
|
if ('unknown' === $result->browser_type) { |
367
|
|
|
throw new UnknownBrowserTypeException('Unknown browser type found'); |
368
|
|
|
} |
369
|
|
|
|
370
|
|
|
if (in_array($result->browser_type, ['Bot/Crawler', 'Library'])) { |
371
|
|
|
return '.'; |
372
|
|
|
} |
373
|
|
|
|
374
|
|
|
if ('unknown' === $result->platform) { |
375
|
|
|
throw new UnknownPlatformException('Unknown platform found'); |
376
|
|
|
} |
377
|
|
|
|
378
|
|
|
if ('unknown' === $result->device_type) { |
379
|
|
|
throw new UnknownDeviceException('Unknown device type found'); |
380
|
|
|
} |
381
|
|
|
|
382
|
|
|
if ('unknown' === $result->renderingengine_name) { |
383
|
|
|
throw new UnknownEngineException('Unknown rendering engine found'); |
384
|
|
|
} |
385
|
|
|
|
386
|
|
|
return '.'; |
387
|
|
|
} |
388
|
|
|
|
389
|
|
|
private function getFiles(InputInterface $input) : Finder |
390
|
|
|
{ |
391
|
|
|
$finder = Finder::create(); |
392
|
|
|
|
393
|
|
|
if ($input->getOption('log-file')) { |
394
|
|
|
$file = $input->getOption('log-file'); |
395
|
|
|
$finder->append(Finder::create()->in(dirname($file))->name(basename($file))); |
396
|
|
|
} |
397
|
|
|
|
398
|
|
|
if ($input->getOption('log-dir')) { |
399
|
|
|
$dirFinder = Finder::create() |
400
|
|
|
->in($input->getOption('log-dir')); |
401
|
|
|
array_map([$dirFinder, 'name'], $input->getOption('include')); |
402
|
|
|
array_map([$dirFinder, 'notName'], $input->getOption('exclude')); |
403
|
|
|
|
404
|
|
|
$finder->append($dirFinder); |
405
|
|
|
} |
406
|
|
|
|
407
|
|
|
return $finder; |
408
|
|
|
} |
409
|
|
|
|
410
|
|
|
private function getPath(SplFileInfo $file) : string |
411
|
|
|
{ |
412
|
|
|
switch ($file->getExtension()) { |
413
|
|
|
case 'gz': |
414
|
|
|
$path = 'compress.zlib://' . $file->getPathname(); |
415
|
|
|
|
416
|
|
|
break; |
417
|
|
|
case 'bz2': |
418
|
|
|
$path = 'compress.bzip2://' . $file->getPathname(); |
419
|
|
|
|
420
|
|
|
break; |
421
|
|
|
default: |
422
|
|
|
$path = $file->getPathname(); |
423
|
|
|
|
424
|
|
|
break; |
425
|
|
|
} |
426
|
|
|
|
427
|
|
|
return $path; |
428
|
|
|
} |
429
|
|
|
} |
430
|
|
|
|
Scrutinizer analyzes your
composer.json
/composer.lock
file if available to determine the classes, and functions that are defined by your dependencies.It seems like the listed class was neither found in your dependencies, nor was it found in the analyzed files in your repository. If you are using some other form of dependency management, you might want to disable this analysis.