We could not synchronize checks via GitHub's checks API since Scrutinizer's GitHub App is not installed for this repository.
1 | <?php |
||||
2 | |||||
3 | /** |
||||
4 | * (c) Kitodo. Key to digital objects e.V. <[email protected]> |
||||
5 | * |
||||
6 | * This file is part of the Kitodo and TYPO3 projects. |
||||
7 | * |
||||
8 | * @license GNU General Public License version 3 or later. |
||||
9 | * For the full copyright and license information, please read the |
||||
10 | * LICENSE.txt file that was distributed with this source code. |
||||
11 | */ |
||||
12 | |||||
13 | namespace Kitodo\Dlf\Command; |
||||
14 | |||||
15 | use Kitodo\Dlf\Common\AbstractDocument; |
||||
16 | use Kitodo\Dlf\Command\BaseCommand; |
||||
17 | use Kitodo\Dlf\Common\Indexer; |
||||
18 | use Kitodo\Dlf\Domain\Model\Document; |
||||
19 | use Symfony\Component\Console\Input\InputInterface; |
||||
20 | use Symfony\Component\Console\Input\InputOption; |
||||
21 | use Symfony\Component\Console\Output\OutputInterface; |
||||
22 | use Symfony\Component\Console\Style\SymfonyStyle; |
||||
23 | use TYPO3\CMS\Core\Utility\GeneralUtility; |
||||
24 | use TYPO3\CMS\Core\Utility\MathUtility; |
||||
25 | use Phpoaipmh\Endpoint; |
||||
26 | use Phpoaipmh\Exception\BaseOaipmhException; |
||||
27 | |||||
28 | /** |
||||
29 | * CLI Command for harvesting OAI-PMH interfaces into database and Solr. |
||||
30 | * |
||||
31 | * @package TYPO3 |
||||
32 | * @subpackage dlf |
||||
33 | * |
||||
34 | * @access public |
||||
35 | */ |
||||
36 | class HarvestCommand extends BaseCommand |
||||
37 | { |
||||
38 | /** |
||||
39 | * Configure the command by defining the name, options and arguments |
||||
40 | * |
||||
41 | * @access public |
||||
42 | * |
||||
43 | * @return void |
||||
44 | */ |
||||
45 | public function configure(): void |
||||
46 | { |
||||
47 | $this |
||||
48 | ->setDescription('Harvest OAI-PMH contents into database and Solr.') |
||||
49 | ->setHelp('') |
||||
50 | ->addOption( |
||||
51 | 'dry-run', |
||||
52 | null, |
||||
53 | InputOption::VALUE_NONE, |
||||
54 | 'If this option is set, the files will not actually be processed but the location URIs are shown.' |
||||
55 | ) |
||||
56 | ->addOption( |
||||
57 | 'lib', |
||||
58 | 'l', |
||||
59 | InputOption::VALUE_REQUIRED, |
||||
60 | 'UID of the library to harvest.' |
||||
61 | ) |
||||
62 | ->addOption( |
||||
63 | 'pid', |
||||
64 | 'p', |
||||
65 | InputOption::VALUE_REQUIRED, |
||||
66 | 'UID of the page the documents should be added to.' |
||||
67 | ) |
||||
68 | ->addOption( |
||||
69 | 'solr', |
||||
70 | 's', |
||||
71 | InputOption::VALUE_REQUIRED, |
||||
72 | '[UID|index_name] of the Solr core the document should be added to.' |
||||
73 | ) |
||||
74 | ->addOption( |
||||
75 | 'from', |
||||
76 | null, |
||||
77 | InputOption::VALUE_OPTIONAL, |
||||
78 | 'Datestamp (YYYY-MM-DD) to begin harvesting from.' |
||||
79 | ) |
||||
80 | ->addOption( |
||||
81 | 'until', |
||||
82 | null, |
||||
83 | InputOption::VALUE_OPTIONAL, |
||||
84 | 'Datestamp (YYYY-MM-DD) to end harvesting on.' |
||||
85 | ) |
||||
86 | ->addOption( |
||||
87 | 'set', |
||||
88 | null, |
||||
89 | InputOption::VALUE_OPTIONAL, |
||||
90 | 'Name of the set to limit harvesting to.' |
||||
91 | ); |
||||
92 | } |
||||
93 | |||||
94 | /** |
||||
95 | * Executes the command to index the given document to DB and SOLR. |
||||
96 | * |
||||
97 | * @access protected |
||||
98 | * |
||||
99 | * @param InputInterface $input The input parameters |
||||
100 | * @param OutputInterface $output The Symfony interface for outputs on console |
||||
101 | * |
||||
102 | * @return int |
||||
103 | */ |
||||
104 | protected function execute(InputInterface $input, OutputInterface $output): int |
||||
105 | { |
||||
106 | $dryRun = $input->getOption('dry-run') != false ? true : false; |
||||
107 | |||||
108 | $io = new SymfonyStyle($input, $output); |
||||
109 | $io->title($this->getDescription()); |
||||
110 | |||||
111 | $this->initializeRepositories($input->getOption('pid')); |
||||
112 | |||||
113 | if ($this->storagePid == 0) { |
||||
114 | $io->error('ERROR: No valid PID (' . $this->storagePid . ') given.'); |
||||
115 | return BaseCommand::FAILURE; |
||||
116 | } |
||||
117 | |||||
118 | if ( |
||||
119 | !empty($input->getOption('solr')) |
||||
120 | && !is_array($input->getOption('solr')) |
||||
121 | ) { |
||||
122 | $allSolrCores = $this->getSolrCores($this->storagePid); |
||||
123 | if (MathUtility::canBeInterpretedAsInteger($input->getOption('solr'))) { |
||||
124 | $solrCoreUid = MathUtility::forceIntegerInRange((int) $input->getOption('solr'), 0); |
||||
125 | } else { |
||||
126 | $solrCoreUid = $allSolrCores[$input->getOption('solr')]; |
||||
127 | } |
||||
128 | // Abort if solrCoreUid is empty or not in the array of allowed solr cores. |
||||
129 | if (empty($solrCoreUid) || !in_array($solrCoreUid, $allSolrCores)) { |
||||
130 | $output_solrCores = []; |
||||
131 | foreach ($allSolrCores as $index_name => $uid) { |
||||
132 | $output_solrCores[] = $uid . ' : ' . $index_name; |
||||
133 | } |
||||
134 | if (empty($output_solrCores)) { |
||||
135 | $io->error('ERROR: No valid Solr core ("' . $input->getOption('solr') . '") given. No valid cores found on PID ' . $this->storagePid . ".\n"); |
||||
136 | return BaseCommand::FAILURE; |
||||
137 | } else { |
||||
138 | $io->error('ERROR: No valid Solr core ("' . $input->getOption('solr') . '") given. ' . "Valid cores are (<uid>:<index_name>):\n" . implode("\n", $output_solrCores) . "\n"); |
||||
139 | return BaseCommand::FAILURE; |
||||
140 | } |
||||
141 | } |
||||
142 | } else { |
||||
143 | $io->error('ERROR: Required parameter --solr|-s is missing or array.'); |
||||
144 | return BaseCommand::FAILURE; |
||||
145 | } |
||||
146 | |||||
147 | if (MathUtility::canBeInterpretedAsInteger($input->getOption('lib'))) { |
||||
148 | $this->owner = $this->libraryRepository->findByUid(MathUtility::forceIntegerInRange((int) $input->getOption('lib'), 1)); |
||||
149 | } |
||||
150 | |||||
151 | if ($this->owner) { |
||||
152 | $baseUrl = $this->owner->getOaiBase(); |
||||
153 | } else { |
||||
154 | $io->error('ERROR: Required parameter --lib|-l is not a valid UID.'); |
||||
155 | return BaseCommand::FAILURE; |
||||
156 | } |
||||
157 | if (!GeneralUtility::isValidUrl($baseUrl)) { |
||||
158 | $io->error('ERROR: No valid OAI Base URL set for library with given UID ("' . $input->getOption('lib') . '").'); |
||||
159 | return BaseCommand::FAILURE; |
||||
160 | } else { |
||||
161 | try { |
||||
162 | $oai = Endpoint::build($baseUrl); |
||||
163 | } catch (BaseoaipmhException $e) { |
||||
164 | $this->handleOaiError($e, $io); |
||||
165 | } |
||||
166 | } |
||||
167 | |||||
168 | if ( |
||||
169 | !is_array($input->getOption('from')) |
||||
170 | && preg_match('/^\d{4}-\d{2}-\d{2}$/', $input->getOption('from')) |
||||
171 | ) { |
||||
172 | $from = new \DateTime($input->getOption('from')); |
||||
173 | } else { |
||||
174 | $from = null; |
||||
175 | } |
||||
176 | |||||
177 | if ( |
||||
178 | !is_array($input->getOption('until')) |
||||
179 | && preg_match('/^\d{4}-\d{2}-\d{2}$/', $input->getOption('until')) |
||||
180 | ) { |
||||
181 | $until = new \DateTime($input->getOption('until')); |
||||
182 | } else { |
||||
183 | $until = null; |
||||
184 | } |
||||
185 | |||||
186 | $set = null; |
||||
187 | if ( |
||||
188 | !is_array($input->getOption('set')) |
||||
189 | && !empty($input->getOption('set')) |
||||
190 | && !empty($oai) |
||||
191 | ) { |
||||
192 | $setsAvailable = $oai->listSets(); |
||||
193 | foreach ($setsAvailable as $setAvailable) { |
||||
194 | if ((string) $setAvailable->setSpec === $input->getOption('set')) { |
||||
195 | $set = $input->getOption('set'); |
||||
196 | break; |
||||
197 | } |
||||
198 | } |
||||
199 | if (empty($set)) { |
||||
200 | $io->error('ERROR: OAI interface does not provide a set with given setSpec ("' . $input->getOption('set') . '").'); |
||||
201 | return BaseCommand::FAILURE; |
||||
202 | } |
||||
203 | } |
||||
204 | |||||
205 | $identifiers = []; |
||||
206 | // Get OAI record identifiers to process. |
||||
207 | try { |
||||
208 | if (!empty($oai)) { |
||||
209 | $identifiers = $oai->listIdentifiers('mets', $from, $until, $set); |
||||
210 | } else { |
||||
211 | $io->error('ERROR: OAI interface does not exist.'); |
||||
212 | } |
||||
213 | } catch (BaseoaipmhException $exception) { |
||||
214 | $this->handleOaiError($exception, $io); |
||||
215 | } |
||||
216 | |||||
217 | // Process all identifiers. |
||||
218 | $baseLocation = $baseUrl . (parse_url($baseUrl, PHP_URL_QUERY) ? '&' : '?'); |
||||
219 | foreach ($identifiers as $identifier) { |
||||
220 | // Build OAI GetRecord URL... |
||||
221 | $params = [ |
||||
222 | 'verb' => 'GetRecord', |
||||
223 | 'metadataPrefix' => 'mets', |
||||
224 | 'identifier' => (string) $identifier->identifier |
||||
225 | ]; |
||||
226 | $docLocation = $baseLocation . http_build_query($params); |
||||
227 | // ...index the document... |
||||
228 | $document = null; |
||||
229 | $doc = AbstractDocument::getInstance($docLocation, ['storagePid' => $this->storagePid], true); |
||||
230 | |||||
231 | if ($doc === null) { |
||||
232 | $io->warning('WARNING: Document "' . $docLocation . '" could not be loaded. Skip to next document.'); |
||||
233 | continue; |
||||
234 | } |
||||
235 | |||||
236 | if ($doc->recordId) { |
||||
237 | $document = $this->documentRepository->findOneByRecordId($doc->recordId); |
||||
238 | } |
||||
239 | |||||
240 | if ($document === null) { |
||||
241 | // create new Document object |
||||
242 | $document = GeneralUtility::makeInstance(Document::class); |
||||
243 | } |
||||
244 | |||||
245 | $document->setLocation($docLocation); |
||||
246 | $document->setSolrcore($solrCoreUid); |
||||
247 | |||||
248 | if ($dryRun) { |
||||
249 | $io->writeln('DRY RUN: Would index ' . $document->getUid() . ' ("' . $document->getLocation() . '") on PID ' . $this->storagePid . ' and Solr core ' . $solrCoreUid . '.'); |
||||
0 ignored issues
–
show
The method
getUid() does not exist on TYPO3\CMS\Extbase\Persistence\QueryResultInterface .
(
Ignorable by Annotation
)
If this is a false-positive, you can also ignore this issue in your code via the
This check looks for calls to methods that do not seem to exist on a given type. It looks for the method on the type itself as well as in inherited classes or implemented interfaces. This is most likely a typographical error or the method has been renamed.
Loading history...
|
|||||
250 | } else { |
||||
251 | if ($io->isVerbose()) { |
||||
252 | $io->writeln(date('Y-m-d H:i:s') . ' Indexing ' . $document->getUid() . ' ("' . $document->getLocation() . '") on PID ' . $this->storagePid . ' and Solr core ' . $solrCoreUid . '.'); |
||||
253 | } |
||||
254 | $document->setCurrentDocument($doc); |
||||
255 | // save to database |
||||
256 | $this->saveToDatabase($document); |
||||
257 | // add to index |
||||
258 | Indexer::add($document, $this->documentRepository); |
||||
259 | } |
||||
260 | } |
||||
261 | |||||
262 | $io->success('All done!'); |
||||
263 | |||||
264 | return BaseCommand::SUCCESS; |
||||
265 | } |
||||
266 | |||||
267 | /** |
||||
268 | * Handles OAI errors |
||||
269 | * |
||||
270 | * @access protected |
||||
271 | * |
||||
272 | * @param BaseoaipmhException $exception Instance of exception thrown |
||||
273 | * @param SymfonyStyle $io |
||||
274 | * |
||||
275 | * @return void |
||||
276 | */ |
||||
277 | protected function handleOaiError(BaseoaipmhException $exception, SymfonyStyle $io): void |
||||
278 | { |
||||
279 | $io->error('ERROR: Trying to retrieve data from OAI interface resulted in error:' . "\n " . $exception->getMessage()); |
||||
280 | } |
||||
281 | } |
||||
282 |
This check looks for calls to methods that do not seem to exist on a given type. It looks for the method on the type itself as well as in inherited classes or implemented interfaces.
This is most likely a typographical error or the method has been renamed.