|
1
|
|
|
<?php |
|
2
|
|
|
|
|
3
|
|
|
declare(strict_types=1); |
|
4
|
|
|
|
|
5
|
|
|
/* |
|
6
|
|
|
* Copyright (c) 2022 Ne-Lexa <[email protected]> |
|
7
|
|
|
* |
|
8
|
|
|
* For the full copyright and license information, please view |
|
9
|
|
|
* the LICENSE file that was distributed with this source code. |
|
10
|
|
|
* |
|
11
|
|
|
* @see https://github.com/Ne-Lexa/roach-php-bundle |
|
12
|
|
|
*/ |
|
13
|
|
|
|
|
14
|
|
|
namespace Nelexa\RoachPhpBundle\Command; |
|
15
|
|
|
|
|
16
|
|
|
use RoachPHP\Roach; |
|
17
|
|
|
use RoachPHP\Spider\Configuration\Overrides; |
|
18
|
|
|
use Symfony\Component\Console\Command\Command; |
|
19
|
|
|
use Symfony\Component\Console\Input\InputArgument; |
|
20
|
|
|
use Symfony\Component\Console\Input\InputInterface; |
|
21
|
|
|
use Symfony\Component\Console\Input\InputOption; |
|
22
|
|
|
use Symfony\Component\Console\Output\OutputInterface; |
|
23
|
|
|
use Symfony\Component\Console\Style\OutputStyle; |
|
24
|
|
|
use Symfony\Component\Console\Style\SymfonyStyle; |
|
25
|
|
|
use Symfony\Component\DependencyInjection\ServiceLocator; |
|
26
|
|
|
use Symfony\Component\Serializer\Encoder\JsonEncode; |
|
27
|
|
|
use Symfony\Component\Serializer\SerializerInterface; |
|
28
|
|
|
|
|
29
|
|
|
final class RunSpiderCommand extends Command |
|
30
|
|
|
{ |
|
31
|
|
|
protected static $defaultName = 'roach:run'; |
|
32
|
|
|
|
|
33
|
|
|
protected static $defaultDescription = 'Run the provided spider'; |
|
34
|
|
|
|
|
35
|
|
|
/** @var array<class-string<\RoachPHP\Spider\SpiderInterface>, array<string>> */ |
|
|
|
|
|
|
36
|
|
|
private array $spiderNames; |
|
37
|
|
|
|
|
38
|
3 |
|
public function __construct(private ServiceLocator $serviceLocator, private SerializerInterface $serializer) |
|
39
|
|
|
{ |
|
40
|
|
|
/** @var array<class-string<\RoachPHP\Spider\SpiderInterface>> $providedServices */ |
|
41
|
3 |
|
$providedServices = $this->serviceLocator->getProvidedServices(); |
|
42
|
3 |
|
$this->spiderNames = $this->buildSpiderNameAliases($providedServices); |
|
43
|
3 |
|
parent::__construct(); |
|
44
|
|
|
} |
|
45
|
|
|
|
|
46
|
3 |
|
protected function configure(): void |
|
47
|
|
|
{ |
|
48
|
3 |
|
$spiderArgDescription = "Spider class name\nSupport spiders:\n"; |
|
49
|
|
|
|
|
50
|
3 |
|
foreach ($this->spiderNames as $className => $aliases) { |
|
51
|
3 |
|
$spiderArgDescription .= '[*] <comment>' . $className . '</comment> or aliases <info>' |
|
52
|
3 |
|
. implode('</info>, <info>', $aliases) |
|
53
|
|
|
. '</info>' . \PHP_EOL; |
|
54
|
|
|
} |
|
55
|
|
|
|
|
56
|
|
|
$this |
|
57
|
3 |
|
->addArgument('spider', InputArgument::OPTIONAL, rtrim($spiderArgDescription)) |
|
58
|
3 |
|
->addOption('delay', 't', InputOption::VALUE_OPTIONAL, 'The delay (in seconds) between requests.') |
|
59
|
3 |
|
->addOption('concurrency', 'p', InputOption::VALUE_OPTIONAL, 'The number of concurrent requests.') |
|
60
|
3 |
|
->addOption('output', 'o', InputOption::VALUE_OPTIONAL, 'Save to JSON file') |
|
61
|
|
|
; |
|
62
|
|
|
} |
|
63
|
|
|
|
|
64
|
|
|
/** |
|
65
|
|
|
* @param array<class-string<\RoachPHP\Spider\SpiderInterface>> $services |
|
|
|
|
|
|
66
|
|
|
* |
|
67
|
|
|
* @return array<class-string<\RoachPHP\Spider\SpiderInterface>, array<string>> |
|
|
|
|
|
|
68
|
|
|
*/ |
|
69
|
3 |
|
private function buildSpiderNameAliases(array $services): array |
|
70
|
|
|
{ |
|
71
|
3 |
|
$aliasServices = []; |
|
72
|
|
|
|
|
73
|
3 |
|
foreach ($services as $className) { |
|
74
|
3 |
|
$aliases = []; |
|
75
|
|
|
|
|
76
|
3 |
|
if (($lastPosDelim = strrpos($className, '\\')) !== false) { |
|
77
|
3 |
|
$shortClassName = substr($className, $lastPosDelim + 1); |
|
78
|
3 |
|
$aliases[] = $shortClassName; |
|
79
|
|
|
} else { |
|
80
|
|
|
$shortClassName = $className; |
|
81
|
|
|
} |
|
82
|
|
|
|
|
83
|
3 |
|
$snakeCaseClass = strtolower(ltrim(preg_replace('/[A-Z]([A-Z](?![a-z]))*/', '_$0', $shortClassName), '_')); |
|
84
|
3 |
|
$aliases[] = $snakeCaseClass; |
|
85
|
|
|
|
|
86
|
3 |
|
if (preg_match('~^(.*?)_spider$~', $snakeCaseClass, $matches)) { |
|
87
|
3 |
|
$aliases[] = $matches[1]; |
|
88
|
|
|
} |
|
89
|
|
|
|
|
90
|
3 |
|
$aliasServices[$className] = $aliases; |
|
91
|
|
|
} |
|
92
|
|
|
|
|
93
|
3 |
|
return $aliasServices; |
|
94
|
|
|
} |
|
95
|
|
|
|
|
96
|
3 |
|
protected function interact(InputInterface $input, OutputInterface $output): void |
|
97
|
|
|
{ |
|
98
|
3 |
|
$spiderName = $input->getArgument('spider'); |
|
99
|
|
|
|
|
100
|
3 |
|
if ($spiderName === null) { |
|
101
|
|
|
$spiderName = $this->selectSpiderClassName(new SymfonyStyle($input, $output)); |
|
102
|
|
|
$input->setArgument('spider', $spiderName); |
|
103
|
|
|
} |
|
104
|
|
|
} |
|
105
|
|
|
|
|
106
|
|
|
private function selectSpiderClassName(OutputStyle $io): string |
|
107
|
|
|
{ |
|
108
|
|
|
return (string) $io->choice('Choose a spider class', array_values($this->serviceLocator->getProvidedServices())); |
|
109
|
|
|
} |
|
110
|
|
|
|
|
111
|
3 |
|
protected function execute(InputInterface $input, OutputInterface $output): int |
|
112
|
|
|
{ |
|
113
|
3 |
|
$io = new SymfonyStyle($input, $output); |
|
114
|
3 |
|
$outputFilename = $input->getOption('output'); |
|
115
|
3 |
|
$spiderName = $input->getArgument('spider'); |
|
116
|
3 |
|
$spiderClassName = $this->findSpiderClass($spiderName); |
|
117
|
|
|
|
|
118
|
3 |
|
if ($spiderClassName === null) { |
|
119
|
1 |
|
\assert($spiderName !== null); |
|
120
|
1 |
|
$io = new SymfonyStyle($input, $output); |
|
121
|
1 |
|
$io->error('Unknown spider ' . $spiderName); |
|
122
|
|
|
|
|
123
|
1 |
|
return self::FAILURE; |
|
124
|
|
|
} |
|
125
|
|
|
|
|
126
|
2 |
|
$delay = $input->getOption('delay'); |
|
127
|
|
|
|
|
128
|
2 |
|
if ($delay !== null) { |
|
129
|
|
|
$delay = max(0, (int) $delay); |
|
130
|
|
|
} |
|
131
|
|
|
|
|
132
|
2 |
|
$concurrency = $input->getOption('concurrency'); |
|
133
|
|
|
|
|
134
|
2 |
|
if ($concurrency !== null) { |
|
135
|
|
|
$concurrency = max(1, (int) $concurrency); |
|
136
|
|
|
} |
|
137
|
|
|
|
|
138
|
2 |
|
$overrides = new Overrides( |
|
139
|
|
|
concurrency: $concurrency, |
|
140
|
|
|
requestDelay: $delay, |
|
141
|
|
|
); |
|
142
|
|
|
|
|
143
|
2 |
|
if ($outputFilename !== null) { |
|
144
|
1 |
|
$collectData = Roach::collectSpider($spiderClassName, $overrides); |
|
145
|
|
|
|
|
146
|
1 |
|
if (!$this->saveCollectData($collectData, $outputFilename, $io)) { |
|
147
|
1 |
|
return self::FAILURE; |
|
148
|
|
|
} |
|
149
|
|
|
} else { |
|
150
|
1 |
|
Roach::startSpider($spiderClassName, $overrides); |
|
151
|
|
|
} |
|
152
|
|
|
|
|
153
|
2 |
|
return self::SUCCESS; |
|
154
|
|
|
} |
|
155
|
|
|
|
|
156
|
|
|
/** |
|
157
|
|
|
* @return class-string<\RoachPHP\Spider\SpiderInterface>|null |
|
|
|
|
|
|
158
|
|
|
*/ |
|
159
|
3 |
|
private function findSpiderClass(?string $spiderName): ?string |
|
160
|
|
|
{ |
|
161
|
3 |
|
if ($spiderName !== null) { |
|
162
|
3 |
|
foreach ($this->spiderNames as $className => $aliases) { |
|
163
|
3 |
|
if ($className === $spiderName || \in_array($spiderName, $aliases, true)) { |
|
164
|
2 |
|
return $className; |
|
165
|
|
|
} |
|
166
|
|
|
} |
|
167
|
|
|
} |
|
168
|
|
|
|
|
169
|
1 |
|
return null; |
|
170
|
|
|
} |
|
171
|
|
|
|
|
172
|
1 |
|
private function saveCollectData(array $collectData, string $outputFilename, SymfonyStyle $io): bool |
|
173
|
|
|
{ |
|
174
|
1 |
|
$content = $this->serializer->serialize($collectData, 'json', [ |
|
175
|
1 |
|
JsonEncode::OPTIONS => \JSON_UNESCAPED_UNICODE | \JSON_PRETTY_PRINT | \JSON_UNESCAPED_LINE_TERMINATORS | \JSON_UNESCAPED_SLASHES | \JSON_THROW_ON_ERROR, |
|
176
|
|
|
]); |
|
177
|
|
|
|
|
178
|
1 |
|
$dirname = \dirname($outputFilename); |
|
179
|
|
|
|
|
180
|
1 |
|
if (!is_dir($dirname) && !mkdir($dirname, 0755, true) && !is_dir($dirname)) { |
|
181
|
|
|
$io->error(sprintf('Directory "%s" was not created', $dirname)); |
|
182
|
|
|
|
|
183
|
|
|
return false; |
|
184
|
|
|
} |
|
185
|
|
|
|
|
186
|
1 |
|
if (file_put_contents($outputFilename, $content) === false) { |
|
187
|
|
|
$io->error(sprintf('An error occurred while saving output to file %s', $dirname)); |
|
188
|
|
|
|
|
189
|
|
|
return false; |
|
190
|
|
|
} |
|
191
|
|
|
|
|
192
|
1 |
|
$io->success(sprintf('Collected data successfully saved to file %s', $outputFilename)); |
|
193
|
|
|
|
|
194
|
1 |
|
return true; |
|
195
|
|
|
} |
|
196
|
|
|
} |
|
197
|
|
|
|
Ne-Lexa /
roach-php-bundle
Loading history...