We could not synchronize checks via GitHub's checks API since Scrutinizer's GitHub App is not installed for this repository.
1 | <?php |
||||||
2 | |||||||
3 | /** |
||||||
4 | * (c) Kitodo. Key to digital objects e.V. <[email protected]> |
||||||
5 | * |
||||||
6 | * This file is part of the Kitodo and TYPO3 projects. |
||||||
7 | * |
||||||
8 | * @license GNU General Public License version 3 or later. |
||||||
9 | * For the full copyright and license information, please read the |
||||||
10 | * LICENSE.txt file that was distributed with this source code. |
||||||
11 | */ |
||||||
12 | |||||||
13 | namespace Kitodo\Dlf\Command; |
||||||
14 | |||||||
15 | use Kitodo\Dlf\Common\AbstractDocument; |
||||||
16 | use Kitodo\Dlf\Command\BaseCommand; |
||||||
17 | use Kitodo\Dlf\Common\Indexer; |
||||||
18 | use Kitodo\Dlf\Domain\Model\Document; |
||||||
19 | use Symfony\Component\Console\Input\InputInterface; |
||||||
20 | use Symfony\Component\Console\Input\InputOption; |
||||||
21 | use Symfony\Component\Console\Output\OutputInterface; |
||||||
22 | use Symfony\Component\Console\Style\SymfonyStyle; |
||||||
23 | use TYPO3\CMS\Core\Utility\GeneralUtility; |
||||||
24 | use TYPO3\CMS\Core\Utility\MathUtility; |
||||||
25 | use Phpoaipmh\Endpoint; |
||||||
26 | use Phpoaipmh\Exception\BaseOaipmhException; |
||||||
27 | |||||||
28 | /** |
||||||
29 | * CLI Command for harvesting OAI-PMH interfaces into database and Solr. |
||||||
30 | * |
||||||
31 | * @package TYPO3 |
||||||
32 | * @subpackage dlf |
||||||
33 | * |
||||||
34 | * @access public |
||||||
35 | */ |
||||||
36 | class HarvestCommand extends BaseCommand |
||||||
37 | { |
||||||
38 | /** |
||||||
39 | * Configure the command by defining the name, options and arguments |
||||||
40 | * |
||||||
41 | * @access public |
||||||
42 | * |
||||||
43 | * @return void |
||||||
44 | */ |
||||||
45 | public function configure(): void |
||||||
46 | { |
||||||
47 | $this |
||||||
48 | ->setDescription('Harvest OAI-PMH contents into database and Solr.') |
||||||
49 | ->setHelp('') |
||||||
50 | ->addOption( |
||||||
51 | 'dry-run', |
||||||
52 | null, |
||||||
53 | InputOption::VALUE_NONE, |
||||||
54 | 'If this option is set, the files will not actually be processed but the location URIs are shown.' |
||||||
55 | ) |
||||||
56 | ->addOption( |
||||||
57 | 'lib', |
||||||
58 | 'l', |
||||||
59 | InputOption::VALUE_REQUIRED, |
||||||
60 | 'UID of the library to harvest.' |
||||||
61 | ) |
||||||
62 | ->addOption( |
||||||
63 | 'pid', |
||||||
64 | 'p', |
||||||
65 | InputOption::VALUE_REQUIRED, |
||||||
66 | 'UID of the page the documents should be added to.' |
||||||
67 | ) |
||||||
68 | ->addOption( |
||||||
69 | 'solr', |
||||||
70 | 's', |
||||||
71 | InputOption::VALUE_REQUIRED, |
||||||
72 | '[UID|index_name] of the Solr core the document should be added to.' |
||||||
73 | ) |
||||||
74 | ->addOption( |
||||||
75 | 'from', |
||||||
76 | null, |
||||||
77 | InputOption::VALUE_OPTIONAL, |
||||||
78 | 'Datestamp (YYYY-MM-DD) to begin harvesting from.' |
||||||
79 | ) |
||||||
80 | ->addOption( |
||||||
81 | 'until', |
||||||
82 | null, |
||||||
83 | InputOption::VALUE_OPTIONAL, |
||||||
84 | 'Datestamp (YYYY-MM-DD) to end harvesting on.' |
||||||
85 | ) |
||||||
86 | ->addOption( |
||||||
87 | 'set', |
||||||
88 | null, |
||||||
89 | InputOption::VALUE_OPTIONAL, |
||||||
90 | 'Name of the set to limit harvesting to.' |
||||||
91 | ); |
||||||
92 | } |
||||||
93 | |||||||
94 | /** |
||||||
95 | * Executes the command to index the given document to DB and SOLR. |
||||||
96 | * |
||||||
97 | * @access protected |
||||||
98 | * |
||||||
99 | * @param InputInterface $input The input parameters |
||||||
100 | * @param OutputInterface $output The Symfony interface for outputs on console |
||||||
101 | * |
||||||
102 | * @return int |
||||||
103 | */ |
||||||
104 | protected function execute(InputInterface $input, OutputInterface $output): int |
||||||
105 | { |
||||||
106 | $dryRun = $input->getOption('dry-run') != false ? true : false; |
||||||
107 | |||||||
108 | $io = new SymfonyStyle($input, $output); |
||||||
109 | $io->title($this->getDescription()); |
||||||
110 | |||||||
111 | $this->initializeRepositories($input->getOption('pid')); |
||||||
112 | |||||||
113 | if ($this->storagePid == 0) { |
||||||
114 | $io->error('ERROR: No valid PID (' . $this->storagePid . ') given.'); |
||||||
115 | return BaseCommand::FAILURE; |
||||||
116 | } |
||||||
117 | |||||||
118 | if ( |
||||||
119 | !empty($input->getOption('solr')) |
||||||
120 | && !is_array($input->getOption('solr')) |
||||||
121 | ) { |
||||||
122 | $allSolrCores = $this->getSolrCores($this->storagePid); |
||||||
123 | if (MathUtility::canBeInterpretedAsInteger($input->getOption('solr'))) { |
||||||
124 | $solrCoreUid = MathUtility::forceIntegerInRange((int) $input->getOption('solr'), 0); |
||||||
125 | } else { |
||||||
126 | $solrCoreUid = $allSolrCores[$input->getOption('solr')]; |
||||||
127 | } |
||||||
128 | // Abort if solrCoreUid is empty or not in the array of allowed solr cores. |
||||||
129 | if (empty($solrCoreUid) || !in_array($solrCoreUid, $allSolrCores)) { |
||||||
130 | $output_solrCores = []; |
||||||
131 | foreach ($allSolrCores as $index_name => $uid) { |
||||||
132 | $output_solrCores[] = $uid . ' : ' . $index_name; |
||||||
133 | } |
||||||
134 | if (empty($output_solrCores)) { |
||||||
135 | $io->error('ERROR: No valid Solr core ("' . $input->getOption('solr') . '") given. No valid cores found on PID ' . $this->storagePid . ".\n"); |
||||||
136 | return BaseCommand::FAILURE; |
||||||
137 | } else { |
||||||
138 | $io->error('ERROR: No valid Solr core ("' . $input->getOption('solr') . '") given. ' . "Valid cores are (<uid>:<index_name>):\n" . implode("\n", $output_solrCores) . "\n"); |
||||||
139 | return BaseCommand::FAILURE; |
||||||
140 | } |
||||||
141 | } |
||||||
142 | } else { |
||||||
143 | $io->error('ERROR: Required parameter --solr|-s is missing or array.'); |
||||||
144 | return BaseCommand::FAILURE; |
||||||
145 | } |
||||||
146 | |||||||
147 | if (MathUtility::canBeInterpretedAsInteger($input->getOption('lib'))) { |
||||||
148 | $this->owner = $this->libraryRepository->findByUid(MathUtility::forceIntegerInRange((int) $input->getOption('lib'), 1)); |
||||||
149 | } |
||||||
150 | |||||||
151 | if ($this->owner) { |
||||||
152 | $baseUrl = $this->owner->getOaiBase(); |
||||||
153 | } else { |
||||||
154 | $io->error('ERROR: Required parameter --lib|-l is not a valid UID.'); |
||||||
155 | return BaseCommand::FAILURE; |
||||||
156 | } |
||||||
157 | if (!GeneralUtility::isValidUrl($baseUrl)) { |
||||||
158 | $io->error('ERROR: No valid OAI Base URL set for library with given UID ("' . $input->getOption('lib') . '").'); |
||||||
159 | return BaseCommand::FAILURE; |
||||||
160 | } else { |
||||||
161 | try { |
||||||
162 | $oai = Endpoint::build($baseUrl); |
||||||
163 | } catch (BaseoaipmhException $e) { |
||||||
164 | $this->handleOaiError($e, $io); |
||||||
165 | } |
||||||
166 | } |
||||||
167 | |||||||
168 | if ( |
||||||
169 | !is_array($input->getOption('from')) |
||||||
170 | && preg_match('/^\d{4}-\d{2}-\d{2}$/', $input->getOption('from')) |
||||||
171 | ) { |
||||||
172 | $from = new \DateTime($input->getOption('from')); |
||||||
173 | } else { |
||||||
174 | $from = null; |
||||||
175 | } |
||||||
176 | |||||||
177 | if ( |
||||||
178 | !is_array($input->getOption('until')) |
||||||
179 | && preg_match('/^\d{4}-\d{2}-\d{2}$/', $input->getOption('until')) |
||||||
180 | ) { |
||||||
181 | $until = new \DateTime($input->getOption('until')); |
||||||
182 | } else { |
||||||
183 | $until = null; |
||||||
184 | } |
||||||
185 | |||||||
186 | $set = null; |
||||||
187 | if ( |
||||||
188 | !is_array($input->getOption('set')) |
||||||
189 | && !empty($input->getOption('set')) |
||||||
190 | && !empty($oai) |
||||||
191 | ) { |
||||||
192 | $setsAvailable = $oai->listSets(); |
||||||
193 | foreach ($setsAvailable as $setAvailable) { |
||||||
194 | if ((string) $setAvailable->setSpec === $input->getOption('set')) { |
||||||
195 | $set = $input->getOption('set'); |
||||||
196 | break; |
||||||
197 | } |
||||||
198 | } |
||||||
199 | if (empty($set)) { |
||||||
200 | $io->error('ERROR: OAI interface does not provide a set with given setSpec ("' . $input->getOption('set') . '").'); |
||||||
201 | return BaseCommand::FAILURE; |
||||||
202 | } |
||||||
203 | } |
||||||
204 | |||||||
205 | $identifiers = []; |
||||||
206 | // Get OAI record identifiers to process. |
||||||
207 | try { |
||||||
208 | if (!empty($oai)) { |
||||||
209 | $identifiers = $oai->listIdentifiers('mets', $from, $until, $set); |
||||||
210 | } else { |
||||||
211 | $io->error('ERROR: OAI interface does not exist.'); |
||||||
212 | } |
||||||
213 | } catch (BaseoaipmhException $exception) { |
||||||
214 | $this->handleOaiError($exception, $io); |
||||||
215 | } |
||||||
216 | |||||||
217 | // Process all identifiers. |
||||||
218 | $baseLocation = $baseUrl . (parse_url($baseUrl, PHP_URL_QUERY) ? '&' : '?'); |
||||||
219 | foreach ($identifiers as $identifier) { |
||||||
220 | // Build OAI GetRecord URL... |
||||||
221 | $params = [ |
||||||
222 | 'verb' => 'GetRecord', |
||||||
223 | 'metadataPrefix' => 'mets', |
||||||
224 | 'identifier' => (string) $identifier->identifier |
||||||
225 | ]; |
||||||
226 | $docLocation = $baseLocation . http_build_query($params); |
||||||
227 | // ...index the document... |
||||||
228 | $document = null; |
||||||
229 | $doc = AbstractDocument::getInstance($docLocation, ['storagePid' => $this->storagePid], true); |
||||||
230 | |||||||
231 | if ($doc === null) { |
||||||
232 | $io->warning('WARNING: Document "' . $docLocation . '" could not be loaded. Skip to next document.'); |
||||||
233 | continue; |
||||||
234 | } |
||||||
235 | |||||||
236 | if ($doc->recordId) { |
||||||
237 | $document = $this->documentRepository->findOneByRecordId($doc->recordId); |
||||||
0 ignored issues
–
show
Bug
introduced
by
Loading history...
|
|||||||
238 | } |
||||||
239 | |||||||
240 | if ($document === null) { |
||||||
241 | // create new Document object |
||||||
242 | $document = GeneralUtility::makeInstance(Document::class); |
||||||
243 | } |
||||||
244 | |||||||
245 | $document->setLocation($docLocation); |
||||||
0 ignored issues
–
show
The method
setLocation() does not exist on TYPO3\CMS\Extbase\Persistence\QueryResultInterface .
(
Ignorable by Annotation
)
If this is a false-positive, you can also ignore this issue in your code via the
This check looks for calls to methods that do not seem to exist on a given type. It looks for the method on the type itself as well as in inherited classes or implemented interfaces. This is most likely a typographical error or the method has been renamed.
Loading history...
|
|||||||
246 | $document->setSolrcore($solrCoreUid); |
||||||
0 ignored issues
–
show
The method
setSolrcore() does not exist on TYPO3\CMS\Extbase\Persistence\QueryResultInterface .
(
Ignorable by Annotation
)
If this is a false-positive, you can also ignore this issue in your code via the
This check looks for calls to methods that do not seem to exist on a given type. It looks for the method on the type itself as well as in inherited classes or implemented interfaces. This is most likely a typographical error or the method has been renamed.
Loading history...
|
|||||||
247 | |||||||
248 | if ($dryRun) { |
||||||
249 | $io->writeln('DRY RUN: Would index ' . $document->getUid() . ' ("' . $document->getLocation() . '") on PID ' . $this->storagePid . ' and Solr core ' . $solrCoreUid . '.'); |
||||||
0 ignored issues
–
show
The method
getLocation() does not exist on TYPO3\CMS\Extbase\Persistence\QueryResultInterface .
(
Ignorable by Annotation
)
If this is a false-positive, you can also ignore this issue in your code via the
This check looks for calls to methods that do not seem to exist on a given type. It looks for the method on the type itself as well as in inherited classes or implemented interfaces. This is most likely a typographical error or the method has been renamed.
Loading history...
The method
getUid() does not exist on TYPO3\CMS\Extbase\Persistence\QueryResultInterface .
(
Ignorable by Annotation
)
If this is a false-positive, you can also ignore this issue in your code via the
This check looks for calls to methods that do not seem to exist on a given type. It looks for the method on the type itself as well as in inherited classes or implemented interfaces. This is most likely a typographical error or the method has been renamed.
Loading history...
|
|||||||
250 | } else { |
||||||
251 | if ($io->isVerbose()) { |
||||||
252 | $io->writeln(date('Y-m-d H:i:s') . ' Indexing ' . $document->getUid() . ' ("' . $document->getLocation() . '") on PID ' . $this->storagePid . ' and Solr core ' . $solrCoreUid . '.'); |
||||||
253 | } |
||||||
254 | $document->setCurrentDocument($doc); |
||||||
0 ignored issues
–
show
The method
setCurrentDocument() does not exist on TYPO3\CMS\Extbase\Persistence\QueryResultInterface .
(
Ignorable by Annotation
)
If this is a false-positive, you can also ignore this issue in your code via the
This check looks for calls to methods that do not seem to exist on a given type. It looks for the method on the type itself as well as in inherited classes or implemented interfaces. This is most likely a typographical error or the method has been renamed.
Loading history...
|
|||||||
255 | // save to database |
||||||
256 | $this->saveToDatabase($document); |
||||||
257 | // add to index |
||||||
258 | Indexer::add($document, $this->documentRepository); |
||||||
259 | } |
||||||
260 | } |
||||||
261 | |||||||
262 | $io->success('All done!'); |
||||||
263 | |||||||
264 | return BaseCommand::SUCCESS; |
||||||
265 | } |
||||||
266 | |||||||
267 | /** |
||||||
268 | * Handles OAI errors |
||||||
269 | * |
||||||
270 | * @access protected |
||||||
271 | * |
||||||
272 | * @param BaseoaipmhException $exception Instance of exception thrown |
||||||
273 | * @param SymfonyStyle $io |
||||||
274 | * |
||||||
275 | * @return void |
||||||
276 | */ |
||||||
277 | protected function handleOaiError(BaseoaipmhException $exception, SymfonyStyle $io): void |
||||||
278 | { |
||||||
279 | $io->error('ERROR: Trying to retrieve data from OAI interface resulted in error:' . "\n " . $exception->getMessage()); |
||||||
280 | } |
||||||
281 | } |
||||||
282 |