@@ -57,423 +57,423 @@ |
||
| 57 | 57 | use function preg_quote; |
| 58 | 58 | |
| 59 | 59 | class ConvertType extends Command implements CompletionAwareInterface { |
| 60 | - /** |
|
| 61 | - * @var \OCP\IConfig |
|
| 62 | - */ |
|
| 63 | - protected $config; |
|
| 64 | - |
|
| 65 | - /** |
|
| 66 | - * @var \OC\DB\ConnectionFactory |
|
| 67 | - */ |
|
| 68 | - protected $connectionFactory; |
|
| 69 | - |
|
| 70 | - /** @var array */ |
|
| 71 | - protected $columnTypes; |
|
| 72 | - |
|
| 73 | - /** |
|
| 74 | - * @param \OCP\IConfig $config |
|
| 75 | - * @param \OC\DB\ConnectionFactory $connectionFactory |
|
| 76 | - */ |
|
| 77 | - public function __construct(IConfig $config, ConnectionFactory $connectionFactory) { |
|
| 78 | - $this->config = $config; |
|
| 79 | - $this->connectionFactory = $connectionFactory; |
|
| 80 | - parent::__construct(); |
|
| 81 | - } |
|
| 82 | - |
|
| 83 | - protected function configure() { |
|
| 84 | - $this |
|
| 85 | - ->setName('db:convert-type') |
|
| 86 | - ->setDescription('Convert the Nextcloud database to the newly configured one') |
|
| 87 | - ->addArgument( |
|
| 88 | - 'type', |
|
| 89 | - InputArgument::REQUIRED, |
|
| 90 | - 'the type of the database to convert to' |
|
| 91 | - ) |
|
| 92 | - ->addArgument( |
|
| 93 | - 'username', |
|
| 94 | - InputArgument::REQUIRED, |
|
| 95 | - 'the username of the database to convert to' |
|
| 96 | - ) |
|
| 97 | - ->addArgument( |
|
| 98 | - 'hostname', |
|
| 99 | - InputArgument::REQUIRED, |
|
| 100 | - 'the hostname of the database to convert to' |
|
| 101 | - ) |
|
| 102 | - ->addArgument( |
|
| 103 | - 'database', |
|
| 104 | - InputArgument::REQUIRED, |
|
| 105 | - 'the name of the database to convert to' |
|
| 106 | - ) |
|
| 107 | - ->addOption( |
|
| 108 | - 'port', |
|
| 109 | - null, |
|
| 110 | - InputOption::VALUE_REQUIRED, |
|
| 111 | - 'the port of the database to convert to' |
|
| 112 | - ) |
|
| 113 | - ->addOption( |
|
| 114 | - 'password', |
|
| 115 | - null, |
|
| 116 | - InputOption::VALUE_REQUIRED, |
|
| 117 | - 'the password of the database to convert to. Will be asked when not specified. Can also be passed via stdin.' |
|
| 118 | - ) |
|
| 119 | - ->addOption( |
|
| 120 | - 'clear-schema', |
|
| 121 | - null, |
|
| 122 | - InputOption::VALUE_NONE, |
|
| 123 | - 'remove all tables from the destination database' |
|
| 124 | - ) |
|
| 125 | - ->addOption( |
|
| 126 | - 'all-apps', |
|
| 127 | - null, |
|
| 128 | - InputOption::VALUE_NONE, |
|
| 129 | - 'whether to create schema for all apps instead of only installed apps' |
|
| 130 | - ) |
|
| 131 | - ->addOption( |
|
| 132 | - 'chunk-size', |
|
| 133 | - null, |
|
| 134 | - InputOption::VALUE_REQUIRED, |
|
| 135 | - 'the maximum number of database rows to handle in a single query, bigger tables will be handled in chunks of this size. Lower this if the process runs out of memory during conversion.', |
|
| 136 | - 1000 |
|
| 137 | - ) |
|
| 138 | - ; |
|
| 139 | - } |
|
| 140 | - |
|
| 141 | - protected function validateInput(InputInterface $input, OutputInterface $output) { |
|
| 142 | - $type = $this->connectionFactory->normalizeType($input->getArgument('type')); |
|
| 143 | - if ($type === 'sqlite3') { |
|
| 144 | - throw new \InvalidArgumentException( |
|
| 145 | - 'Converting to SQLite (sqlite3) is currently not supported.' |
|
| 146 | - ); |
|
| 147 | - } |
|
| 148 | - if ($type === $this->config->getSystemValue('dbtype', '')) { |
|
| 149 | - throw new \InvalidArgumentException(sprintf( |
|
| 150 | - 'Can not convert from %1$s to %1$s.', |
|
| 151 | - $type |
|
| 152 | - )); |
|
| 153 | - } |
|
| 154 | - if ($type === 'oci' && $input->getOption('clear-schema')) { |
|
| 155 | - // Doctrine unconditionally tries (at least in version 2.3) |
|
| 156 | - // to drop sequence triggers when dropping a table, even though |
|
| 157 | - // such triggers may not exist. This results in errors like |
|
| 158 | - // "ORA-04080: trigger 'OC_STORAGES_AI_PK' does not exist". |
|
| 159 | - throw new \InvalidArgumentException( |
|
| 160 | - 'The --clear-schema option is not supported when converting to Oracle (oci).' |
|
| 161 | - ); |
|
| 162 | - } |
|
| 163 | - } |
|
| 164 | - |
|
| 165 | - protected function readPassword(InputInterface $input, OutputInterface $output) { |
|
| 166 | - // Explicitly specified password |
|
| 167 | - if ($input->getOption('password')) { |
|
| 168 | - return; |
|
| 169 | - } |
|
| 170 | - |
|
| 171 | - // Read from stdin. stream_set_blocking is used to prevent blocking |
|
| 172 | - // when nothing is passed via stdin. |
|
| 173 | - stream_set_blocking(STDIN, 0); |
|
| 174 | - $password = file_get_contents('php://stdin'); |
|
| 175 | - stream_set_blocking(STDIN, 1); |
|
| 176 | - if (trim($password) !== '') { |
|
| 177 | - $input->setOption('password', $password); |
|
| 178 | - return; |
|
| 179 | - } |
|
| 180 | - |
|
| 181 | - // Read password by interacting |
|
| 182 | - if ($input->isInteractive()) { |
|
| 183 | - /** @var QuestionHelper $helper */ |
|
| 184 | - $helper = $this->getHelper('question'); |
|
| 185 | - $question = new Question('What is the database password?'); |
|
| 186 | - $question->setHidden(true); |
|
| 187 | - $question->setHiddenFallback(false); |
|
| 188 | - $password = $helper->ask($input, $output, $question); |
|
| 189 | - $input->setOption('password', $password); |
|
| 190 | - return; |
|
| 191 | - } |
|
| 192 | - } |
|
| 193 | - |
|
| 194 | - protected function execute(InputInterface $input, OutputInterface $output): int { |
|
| 195 | - $this->validateInput($input, $output); |
|
| 196 | - $this->readPassword($input, $output); |
|
| 197 | - |
|
| 198 | - /** @var Connection $fromDB */ |
|
| 199 | - $fromDB = \OC::$server->get(Connection::class); |
|
| 200 | - $toDB = $this->getToDBConnection($input, $output); |
|
| 201 | - |
|
| 202 | - if ($input->getOption('clear-schema')) { |
|
| 203 | - $this->clearSchema($toDB, $input, $output); |
|
| 204 | - } |
|
| 205 | - |
|
| 206 | - $this->createSchema($fromDB, $toDB, $input, $output); |
|
| 207 | - |
|
| 208 | - $toTables = $this->getTables($toDB); |
|
| 209 | - $fromTables = $this->getTables($fromDB); |
|
| 210 | - |
|
| 211 | - // warn/fail if there are more tables in 'from' database |
|
| 212 | - $extraFromTables = array_diff($fromTables, $toTables); |
|
| 213 | - if (!empty($extraFromTables)) { |
|
| 214 | - $output->writeln('<comment>The following tables will not be converted:</comment>'); |
|
| 215 | - $output->writeln($extraFromTables); |
|
| 216 | - if (!$input->getOption('all-apps')) { |
|
| 217 | - $output->writeln('<comment>Please note that tables belonging to available but currently not installed apps</comment>'); |
|
| 218 | - $output->writeln('<comment>can be included by specifying the --all-apps option.</comment>'); |
|
| 219 | - } |
|
| 220 | - |
|
| 221 | - $continueConversion = !$input->isInteractive(); // assume yes for --no-interaction and no otherwise. |
|
| 222 | - $question = new ConfirmationQuestion('Continue with the conversion (y/n)? [n] ', $continueConversion); |
|
| 223 | - |
|
| 224 | - /** @var QuestionHelper $helper */ |
|
| 225 | - $helper = $this->getHelper('question'); |
|
| 226 | - |
|
| 227 | - if (!$helper->ask($input, $output, $question)) { |
|
| 228 | - return 1; |
|
| 229 | - } |
|
| 230 | - } |
|
| 231 | - $intersectingTables = array_intersect($toTables, $fromTables); |
|
| 232 | - $this->convertDB($fromDB, $toDB, $intersectingTables, $input, $output); |
|
| 233 | - return 0; |
|
| 234 | - } |
|
| 235 | - |
|
| 236 | - protected function createSchema(Connection $fromDB, Connection $toDB, InputInterface $input, OutputInterface $output) { |
|
| 237 | - $output->writeln('<info>Creating schema in new database</info>'); |
|
| 238 | - |
|
| 239 | - $fromMS = new MigrationService('core', $fromDB); |
|
| 240 | - $currentMigration = $fromMS->getMigration('current'); |
|
| 241 | - if ($currentMigration !== '0') { |
|
| 242 | - $toMS = new MigrationService('core', $toDB); |
|
| 243 | - $toMS->migrate($currentMigration); |
|
| 244 | - } |
|
| 245 | - |
|
| 246 | - $schemaManager = new \OC\DB\MDB2SchemaManager($toDB); |
|
| 247 | - $apps = $input->getOption('all-apps') ? \OC_App::getAllApps() : \OC_App::getEnabledApps(); |
|
| 248 | - foreach ($apps as $app) { |
|
| 249 | - $output->writeln('<info> - '.$app.'</info>'); |
|
| 250 | - if (file_exists(\OC_App::getAppPath($app).'/appinfo/database.xml')) { |
|
| 251 | - $schemaManager->createDbFromStructure(\OC_App::getAppPath($app).'/appinfo/database.xml'); |
|
| 252 | - } else { |
|
| 253 | - // Make sure autoloading works... |
|
| 254 | - \OC_App::loadApp($app); |
|
| 255 | - $fromMS = new MigrationService($app, $fromDB); |
|
| 256 | - $currentMigration = $fromMS->getMigration('current'); |
|
| 257 | - if ($currentMigration !== '0') { |
|
| 258 | - $toMS = new MigrationService($app, $toDB); |
|
| 259 | - $toMS->migrate($currentMigration, true); |
|
| 260 | - } |
|
| 261 | - } |
|
| 262 | - } |
|
| 263 | - } |
|
| 264 | - |
|
| 265 | - protected function getToDBConnection(InputInterface $input, OutputInterface $output) { |
|
| 266 | - $type = $input->getArgument('type'); |
|
| 267 | - $connectionParams = $this->connectionFactory->createConnectionParams(); |
|
| 268 | - $connectionParams = array_merge($connectionParams, [ |
|
| 269 | - 'host' => $input->getArgument('hostname'), |
|
| 270 | - 'user' => $input->getArgument('username'), |
|
| 271 | - 'password' => $input->getOption('password'), |
|
| 272 | - 'dbname' => $input->getArgument('database'), |
|
| 273 | - ]); |
|
| 274 | - if ($input->getOption('port')) { |
|
| 275 | - $connectionParams['port'] = $input->getOption('port'); |
|
| 276 | - } |
|
| 277 | - return $this->connectionFactory->getConnection($type, $connectionParams); |
|
| 278 | - } |
|
| 279 | - |
|
| 280 | - protected function clearSchema(Connection $db, InputInterface $input, OutputInterface $output) { |
|
| 281 | - $toTables = $this->getTables($db); |
|
| 282 | - if (!empty($toTables)) { |
|
| 283 | - $output->writeln('<info>Clearing schema in new database</info>'); |
|
| 284 | - } |
|
| 285 | - foreach ($toTables as $table) { |
|
| 286 | - $db->getSchemaManager()->dropTable($table); |
|
| 287 | - } |
|
| 288 | - } |
|
| 289 | - |
|
| 290 | - protected function getTables(Connection $db) { |
|
| 291 | - $db->getConfiguration()->setSchemaAssetsFilter(function ($asset) { |
|
| 292 | - /** @var string|AbstractAsset $asset */ |
|
| 293 | - $filterExpression = '/^' . preg_quote($this->config->getSystemValue('dbtableprefix', 'oc_')) . '/'; |
|
| 294 | - if ($asset instanceof AbstractAsset) { |
|
| 295 | - return preg_match($filterExpression, $asset->getName()) !== false; |
|
| 296 | - } |
|
| 297 | - return preg_match($filterExpression, $asset) !== false; |
|
| 298 | - }); |
|
| 299 | - return $db->getSchemaManager()->listTableNames(); |
|
| 300 | - } |
|
| 301 | - |
|
| 302 | - /** |
|
| 303 | - * @param Connection $fromDB |
|
| 304 | - * @param Connection $toDB |
|
| 305 | - * @param Table $table |
|
| 306 | - * @param InputInterface $input |
|
| 307 | - * @param OutputInterface $output |
|
| 308 | - */ |
|
| 309 | - protected function copyTable(Connection $fromDB, Connection $toDB, Table $table, InputInterface $input, OutputInterface $output) { |
|
| 310 | - if ($table->getName() === $toDB->getPrefix() . 'migrations') { |
|
| 311 | - $output->writeln('<comment>Skipping migrations table because it was already filled by running the migrations</comment>'); |
|
| 312 | - return; |
|
| 313 | - } |
|
| 314 | - |
|
| 315 | - $chunkSize = $input->getOption('chunk-size'); |
|
| 316 | - |
|
| 317 | - $query = $fromDB->getQueryBuilder(); |
|
| 318 | - $query->automaticTablePrefix(false); |
|
| 319 | - $query->select($query->func()->count('*', 'num_entries')) |
|
| 320 | - ->from($table->getName()); |
|
| 321 | - $result = $query->execute(); |
|
| 322 | - $count = $result->fetchOne(); |
|
| 323 | - $result->closeCursor(); |
|
| 324 | - |
|
| 325 | - $numChunks = ceil($count / $chunkSize); |
|
| 326 | - if ($numChunks > 1) { |
|
| 327 | - $output->writeln('chunked query, ' . $numChunks . ' chunks'); |
|
| 328 | - } |
|
| 329 | - |
|
| 330 | - $progress = new ProgressBar($output, $count); |
|
| 331 | - $progress->setFormat('very_verbose'); |
|
| 332 | - $progress->start(); |
|
| 333 | - $redraw = $count > $chunkSize ? 100 : ($count > 100 ? 5 : 1); |
|
| 334 | - $progress->setRedrawFrequency($redraw); |
|
| 335 | - |
|
| 336 | - $query = $fromDB->getQueryBuilder(); |
|
| 337 | - $query->automaticTablePrefix(false); |
|
| 338 | - $query->select('*') |
|
| 339 | - ->from($table->getName()) |
|
| 340 | - ->setMaxResults($chunkSize); |
|
| 341 | - |
|
| 342 | - try { |
|
| 343 | - $orderColumns = $table->getPrimaryKeyColumns(); |
|
| 344 | - } catch (Exception $e) { |
|
| 345 | - $orderColumns = []; |
|
| 346 | - foreach ($table->getColumns() as $column) { |
|
| 347 | - $orderColumns[] = $column->getName(); |
|
| 348 | - } |
|
| 349 | - } |
|
| 350 | - |
|
| 351 | - foreach ($orderColumns as $column) { |
|
| 352 | - $query->addOrderBy($column); |
|
| 353 | - } |
|
| 354 | - |
|
| 355 | - $insertQuery = $toDB->getQueryBuilder(); |
|
| 356 | - $insertQuery->automaticTablePrefix(false); |
|
| 357 | - $insertQuery->insert($table->getName()); |
|
| 358 | - $parametersCreated = false; |
|
| 359 | - |
|
| 360 | - for ($chunk = 0; $chunk < $numChunks; $chunk++) { |
|
| 361 | - $query->setFirstResult($chunk * $chunkSize); |
|
| 362 | - |
|
| 363 | - $result = $query->execute(); |
|
| 364 | - |
|
| 365 | - while ($row = $result->fetch()) { |
|
| 366 | - $progress->advance(); |
|
| 367 | - if (!$parametersCreated) { |
|
| 368 | - foreach ($row as $key => $value) { |
|
| 369 | - $insertQuery->setValue($key, $insertQuery->createParameter($key)); |
|
| 370 | - } |
|
| 371 | - $parametersCreated = true; |
|
| 372 | - } |
|
| 373 | - |
|
| 374 | - foreach ($row as $key => $value) { |
|
| 375 | - $type = $this->getColumnType($table, $key); |
|
| 376 | - if ($type !== false) { |
|
| 377 | - $insertQuery->setParameter($key, $value, $type); |
|
| 378 | - } else { |
|
| 379 | - $insertQuery->setParameter($key, $value); |
|
| 380 | - } |
|
| 381 | - } |
|
| 382 | - $insertQuery->execute(); |
|
| 383 | - } |
|
| 384 | - $result->closeCursor(); |
|
| 385 | - } |
|
| 386 | - $progress->finish(); |
|
| 387 | - $output->writeln(''); |
|
| 388 | - } |
|
| 389 | - |
|
| 390 | - protected function getColumnType(Table $table, $columnName) { |
|
| 391 | - $tableName = $table->getName(); |
|
| 392 | - if (isset($this->columnTypes[$tableName][$columnName])) { |
|
| 393 | - return $this->columnTypes[$tableName][$columnName]; |
|
| 394 | - } |
|
| 395 | - |
|
| 396 | - $type = $table->getColumn($columnName)->getType()->getName(); |
|
| 397 | - |
|
| 398 | - switch ($type) { |
|
| 399 | - case Types::BLOB: |
|
| 400 | - case Types::TEXT: |
|
| 401 | - $this->columnTypes[$tableName][$columnName] = IQueryBuilder::PARAM_LOB; |
|
| 402 | - break; |
|
| 403 | - case Types::BOOLEAN: |
|
| 404 | - $this->columnTypes[$tableName][$columnName] = IQueryBuilder::PARAM_BOOL; |
|
| 405 | - break; |
|
| 406 | - default: |
|
| 407 | - $this->columnTypes[$tableName][$columnName] = false; |
|
| 408 | - } |
|
| 409 | - |
|
| 410 | - return $this->columnTypes[$tableName][$columnName]; |
|
| 411 | - } |
|
| 412 | - |
|
| 413 | - protected function convertDB(Connection $fromDB, Connection $toDB, array $tables, InputInterface $input, OutputInterface $output) { |
|
| 414 | - $this->config->setSystemValue('maintenance', true); |
|
| 415 | - $schema = $fromDB->createSchema(); |
|
| 416 | - |
|
| 417 | - try { |
|
| 418 | - // copy table rows |
|
| 419 | - foreach ($tables as $table) { |
|
| 420 | - $output->writeln('<info> - '.$table.'</info>'); |
|
| 421 | - $this->copyTable($fromDB, $toDB, $schema->getTable($table), $input, $output); |
|
| 422 | - } |
|
| 423 | - if ($input->getArgument('type') === 'pgsql') { |
|
| 424 | - $tools = new \OC\DB\PgSqlTools($this->config); |
|
| 425 | - $tools->resynchronizeDatabaseSequences($toDB); |
|
| 426 | - } |
|
| 427 | - // save new database config |
|
| 428 | - $this->saveDBInfo($input); |
|
| 429 | - } catch (\Exception $e) { |
|
| 430 | - $this->config->setSystemValue('maintenance', false); |
|
| 431 | - throw $e; |
|
| 432 | - } |
|
| 433 | - $this->config->setSystemValue('maintenance', false); |
|
| 434 | - } |
|
| 435 | - |
|
| 436 | - protected function saveDBInfo(InputInterface $input) { |
|
| 437 | - $type = $input->getArgument('type'); |
|
| 438 | - $username = $input->getArgument('username'); |
|
| 439 | - $dbHost = $input->getArgument('hostname'); |
|
| 440 | - $dbName = $input->getArgument('database'); |
|
| 441 | - $password = $input->getOption('password'); |
|
| 442 | - if ($input->getOption('port')) { |
|
| 443 | - $dbHost .= ':'.$input->getOption('port'); |
|
| 444 | - } |
|
| 445 | - |
|
| 446 | - $this->config->setSystemValues([ |
|
| 447 | - 'dbtype' => $type, |
|
| 448 | - 'dbname' => $dbName, |
|
| 449 | - 'dbhost' => $dbHost, |
|
| 450 | - 'dbuser' => $username, |
|
| 451 | - 'dbpassword' => $password, |
|
| 452 | - ]); |
|
| 453 | - } |
|
| 454 | - |
|
| 455 | - /** |
|
| 456 | - * Return possible values for the named option |
|
| 457 | - * |
|
| 458 | - * @param string $optionName |
|
| 459 | - * @param CompletionContext $context |
|
| 460 | - * @return string[] |
|
| 461 | - */ |
|
| 462 | - public function completeOptionValues($optionName, CompletionContext $context) { |
|
| 463 | - return []; |
|
| 464 | - } |
|
| 465 | - |
|
| 466 | - /** |
|
| 467 | - * Return possible values for the named argument |
|
| 468 | - * |
|
| 469 | - * @param string $argumentName |
|
| 470 | - * @param CompletionContext $context |
|
| 471 | - * @return string[] |
|
| 472 | - */ |
|
| 473 | - public function completeArgumentValues($argumentName, CompletionContext $context) { |
|
| 474 | - if ($argumentName === 'type') { |
|
| 475 | - return ['mysql', 'oci', 'pgsql']; |
|
| 476 | - } |
|
| 477 | - return []; |
|
| 478 | - } |
|
| 60 | + /** |
|
| 61 | + * @var \OCP\IConfig |
|
| 62 | + */ |
|
| 63 | + protected $config; |
|
| 64 | + |
|
| 65 | + /** |
|
| 66 | + * @var \OC\DB\ConnectionFactory |
|
| 67 | + */ |
|
| 68 | + protected $connectionFactory; |
|
| 69 | + |
|
| 70 | + /** @var array */ |
|
| 71 | + protected $columnTypes; |
|
| 72 | + |
|
| 73 | + /** |
|
| 74 | + * @param \OCP\IConfig $config |
|
| 75 | + * @param \OC\DB\ConnectionFactory $connectionFactory |
|
| 76 | + */ |
|
| 77 | + public function __construct(IConfig $config, ConnectionFactory $connectionFactory) { |
|
| 78 | + $this->config = $config; |
|
| 79 | + $this->connectionFactory = $connectionFactory; |
|
| 80 | + parent::__construct(); |
|
| 81 | + } |
|
| 82 | + |
|
| 83 | + protected function configure() { |
|
| 84 | + $this |
|
| 85 | + ->setName('db:convert-type') |
|
| 86 | + ->setDescription('Convert the Nextcloud database to the newly configured one') |
|
| 87 | + ->addArgument( |
|
| 88 | + 'type', |
|
| 89 | + InputArgument::REQUIRED, |
|
| 90 | + 'the type of the database to convert to' |
|
| 91 | + ) |
|
| 92 | + ->addArgument( |
|
| 93 | + 'username', |
|
| 94 | + InputArgument::REQUIRED, |
|
| 95 | + 'the username of the database to convert to' |
|
| 96 | + ) |
|
| 97 | + ->addArgument( |
|
| 98 | + 'hostname', |
|
| 99 | + InputArgument::REQUIRED, |
|
| 100 | + 'the hostname of the database to convert to' |
|
| 101 | + ) |
|
| 102 | + ->addArgument( |
|
| 103 | + 'database', |
|
| 104 | + InputArgument::REQUIRED, |
|
| 105 | + 'the name of the database to convert to' |
|
| 106 | + ) |
|
| 107 | + ->addOption( |
|
| 108 | + 'port', |
|
| 109 | + null, |
|
| 110 | + InputOption::VALUE_REQUIRED, |
|
| 111 | + 'the port of the database to convert to' |
|
| 112 | + ) |
|
| 113 | + ->addOption( |
|
| 114 | + 'password', |
|
| 115 | + null, |
|
| 116 | + InputOption::VALUE_REQUIRED, |
|
| 117 | + 'the password of the database to convert to. Will be asked when not specified. Can also be passed via stdin.' |
|
| 118 | + ) |
|
| 119 | + ->addOption( |
|
| 120 | + 'clear-schema', |
|
| 121 | + null, |
|
| 122 | + InputOption::VALUE_NONE, |
|
| 123 | + 'remove all tables from the destination database' |
|
| 124 | + ) |
|
| 125 | + ->addOption( |
|
| 126 | + 'all-apps', |
|
| 127 | + null, |
|
| 128 | + InputOption::VALUE_NONE, |
|
| 129 | + 'whether to create schema for all apps instead of only installed apps' |
|
| 130 | + ) |
|
| 131 | + ->addOption( |
|
| 132 | + 'chunk-size', |
|
| 133 | + null, |
|
| 134 | + InputOption::VALUE_REQUIRED, |
|
| 135 | + 'the maximum number of database rows to handle in a single query, bigger tables will be handled in chunks of this size. Lower this if the process runs out of memory during conversion.', |
|
| 136 | + 1000 |
|
| 137 | + ) |
|
| 138 | + ; |
|
| 139 | + } |
|
| 140 | + |
|
| 141 | + protected function validateInput(InputInterface $input, OutputInterface $output) { |
|
| 142 | + $type = $this->connectionFactory->normalizeType($input->getArgument('type')); |
|
| 143 | + if ($type === 'sqlite3') { |
|
| 144 | + throw new \InvalidArgumentException( |
|
| 145 | + 'Converting to SQLite (sqlite3) is currently not supported.' |
|
| 146 | + ); |
|
| 147 | + } |
|
| 148 | + if ($type === $this->config->getSystemValue('dbtype', '')) { |
|
| 149 | + throw new \InvalidArgumentException(sprintf( |
|
| 150 | + 'Can not convert from %1$s to %1$s.', |
|
| 151 | + $type |
|
| 152 | + )); |
|
| 153 | + } |
|
| 154 | + if ($type === 'oci' && $input->getOption('clear-schema')) { |
|
| 155 | + // Doctrine unconditionally tries (at least in version 2.3) |
|
| 156 | + // to drop sequence triggers when dropping a table, even though |
|
| 157 | + // such triggers may not exist. This results in errors like |
|
| 158 | + // "ORA-04080: trigger 'OC_STORAGES_AI_PK' does not exist". |
|
| 159 | + throw new \InvalidArgumentException( |
|
| 160 | + 'The --clear-schema option is not supported when converting to Oracle (oci).' |
|
| 161 | + ); |
|
| 162 | + } |
|
| 163 | + } |
|
| 164 | + |
|
| 165 | + protected function readPassword(InputInterface $input, OutputInterface $output) { |
|
| 166 | + // Explicitly specified password |
|
| 167 | + if ($input->getOption('password')) { |
|
| 168 | + return; |
|
| 169 | + } |
|
| 170 | + |
|
| 171 | + // Read from stdin. stream_set_blocking is used to prevent blocking |
|
| 172 | + // when nothing is passed via stdin. |
|
| 173 | + stream_set_blocking(STDIN, 0); |
|
| 174 | + $password = file_get_contents('php://stdin'); |
|
| 175 | + stream_set_blocking(STDIN, 1); |
|
| 176 | + if (trim($password) !== '') { |
|
| 177 | + $input->setOption('password', $password); |
|
| 178 | + return; |
|
| 179 | + } |
|
| 180 | + |
|
| 181 | + // Read password by interacting |
|
| 182 | + if ($input->isInteractive()) { |
|
| 183 | + /** @var QuestionHelper $helper */ |
|
| 184 | + $helper = $this->getHelper('question'); |
|
| 185 | + $question = new Question('What is the database password?'); |
|
| 186 | + $question->setHidden(true); |
|
| 187 | + $question->setHiddenFallback(false); |
|
| 188 | + $password = $helper->ask($input, $output, $question); |
|
| 189 | + $input->setOption('password', $password); |
|
| 190 | + return; |
|
| 191 | + } |
|
| 192 | + } |
|
| 193 | + |
|
| 194 | + protected function execute(InputInterface $input, OutputInterface $output): int { |
|
| 195 | + $this->validateInput($input, $output); |
|
| 196 | + $this->readPassword($input, $output); |
|
| 197 | + |
|
| 198 | + /** @var Connection $fromDB */ |
|
| 199 | + $fromDB = \OC::$server->get(Connection::class); |
|
| 200 | + $toDB = $this->getToDBConnection($input, $output); |
|
| 201 | + |
|
| 202 | + if ($input->getOption('clear-schema')) { |
|
| 203 | + $this->clearSchema($toDB, $input, $output); |
|
| 204 | + } |
|
| 205 | + |
|
| 206 | + $this->createSchema($fromDB, $toDB, $input, $output); |
|
| 207 | + |
|
| 208 | + $toTables = $this->getTables($toDB); |
|
| 209 | + $fromTables = $this->getTables($fromDB); |
|
| 210 | + |
|
| 211 | + // warn/fail if there are more tables in 'from' database |
|
| 212 | + $extraFromTables = array_diff($fromTables, $toTables); |
|
| 213 | + if (!empty($extraFromTables)) { |
|
| 214 | + $output->writeln('<comment>The following tables will not be converted:</comment>'); |
|
| 215 | + $output->writeln($extraFromTables); |
|
| 216 | + if (!$input->getOption('all-apps')) { |
|
| 217 | + $output->writeln('<comment>Please note that tables belonging to available but currently not installed apps</comment>'); |
|
| 218 | + $output->writeln('<comment>can be included by specifying the --all-apps option.</comment>'); |
|
| 219 | + } |
|
| 220 | + |
|
| 221 | + $continueConversion = !$input->isInteractive(); // assume yes for --no-interaction and no otherwise. |
|
| 222 | + $question = new ConfirmationQuestion('Continue with the conversion (y/n)? [n] ', $continueConversion); |
|
| 223 | + |
|
| 224 | + /** @var QuestionHelper $helper */ |
|
| 225 | + $helper = $this->getHelper('question'); |
|
| 226 | + |
|
| 227 | + if (!$helper->ask($input, $output, $question)) { |
|
| 228 | + return 1; |
|
| 229 | + } |
|
| 230 | + } |
|
| 231 | + $intersectingTables = array_intersect($toTables, $fromTables); |
|
| 232 | + $this->convertDB($fromDB, $toDB, $intersectingTables, $input, $output); |
|
| 233 | + return 0; |
|
| 234 | + } |
|
| 235 | + |
|
| 236 | + protected function createSchema(Connection $fromDB, Connection $toDB, InputInterface $input, OutputInterface $output) { |
|
| 237 | + $output->writeln('<info>Creating schema in new database</info>'); |
|
| 238 | + |
|
| 239 | + $fromMS = new MigrationService('core', $fromDB); |
|
| 240 | + $currentMigration = $fromMS->getMigration('current'); |
|
| 241 | + if ($currentMigration !== '0') { |
|
| 242 | + $toMS = new MigrationService('core', $toDB); |
|
| 243 | + $toMS->migrate($currentMigration); |
|
| 244 | + } |
|
| 245 | + |
|
| 246 | + $schemaManager = new \OC\DB\MDB2SchemaManager($toDB); |
|
| 247 | + $apps = $input->getOption('all-apps') ? \OC_App::getAllApps() : \OC_App::getEnabledApps(); |
|
| 248 | + foreach ($apps as $app) { |
|
| 249 | + $output->writeln('<info> - '.$app.'</info>'); |
|
| 250 | + if (file_exists(\OC_App::getAppPath($app).'/appinfo/database.xml')) { |
|
| 251 | + $schemaManager->createDbFromStructure(\OC_App::getAppPath($app).'/appinfo/database.xml'); |
|
| 252 | + } else { |
|
| 253 | + // Make sure autoloading works... |
|
| 254 | + \OC_App::loadApp($app); |
|
| 255 | + $fromMS = new MigrationService($app, $fromDB); |
|
| 256 | + $currentMigration = $fromMS->getMigration('current'); |
|
| 257 | + if ($currentMigration !== '0') { |
|
| 258 | + $toMS = new MigrationService($app, $toDB); |
|
| 259 | + $toMS->migrate($currentMigration, true); |
|
| 260 | + } |
|
| 261 | + } |
|
| 262 | + } |
|
| 263 | + } |
|
| 264 | + |
|
| 265 | + protected function getToDBConnection(InputInterface $input, OutputInterface $output) { |
|
| 266 | + $type = $input->getArgument('type'); |
|
| 267 | + $connectionParams = $this->connectionFactory->createConnectionParams(); |
|
| 268 | + $connectionParams = array_merge($connectionParams, [ |
|
| 269 | + 'host' => $input->getArgument('hostname'), |
|
| 270 | + 'user' => $input->getArgument('username'), |
|
| 271 | + 'password' => $input->getOption('password'), |
|
| 272 | + 'dbname' => $input->getArgument('database'), |
|
| 273 | + ]); |
|
| 274 | + if ($input->getOption('port')) { |
|
| 275 | + $connectionParams['port'] = $input->getOption('port'); |
|
| 276 | + } |
|
| 277 | + return $this->connectionFactory->getConnection($type, $connectionParams); |
|
| 278 | + } |
|
| 279 | + |
|
| 280 | + protected function clearSchema(Connection $db, InputInterface $input, OutputInterface $output) { |
|
| 281 | + $toTables = $this->getTables($db); |
|
| 282 | + if (!empty($toTables)) { |
|
| 283 | + $output->writeln('<info>Clearing schema in new database</info>'); |
|
| 284 | + } |
|
| 285 | + foreach ($toTables as $table) { |
|
| 286 | + $db->getSchemaManager()->dropTable($table); |
|
| 287 | + } |
|
| 288 | + } |
|
| 289 | + |
|
| 290 | + protected function getTables(Connection $db) { |
|
| 291 | + $db->getConfiguration()->setSchemaAssetsFilter(function ($asset) { |
|
| 292 | + /** @var string|AbstractAsset $asset */ |
|
| 293 | + $filterExpression = '/^' . preg_quote($this->config->getSystemValue('dbtableprefix', 'oc_')) . '/'; |
|
| 294 | + if ($asset instanceof AbstractAsset) { |
|
| 295 | + return preg_match($filterExpression, $asset->getName()) !== false; |
|
| 296 | + } |
|
| 297 | + return preg_match($filterExpression, $asset) !== false; |
|
| 298 | + }); |
|
| 299 | + return $db->getSchemaManager()->listTableNames(); |
|
| 300 | + } |
|
| 301 | + |
|
| 302 | + /** |
|
| 303 | + * @param Connection $fromDB |
|
| 304 | + * @param Connection $toDB |
|
| 305 | + * @param Table $table |
|
| 306 | + * @param InputInterface $input |
|
| 307 | + * @param OutputInterface $output |
|
| 308 | + */ |
|
| 309 | + protected function copyTable(Connection $fromDB, Connection $toDB, Table $table, InputInterface $input, OutputInterface $output) { |
|
| 310 | + if ($table->getName() === $toDB->getPrefix() . 'migrations') { |
|
| 311 | + $output->writeln('<comment>Skipping migrations table because it was already filled by running the migrations</comment>'); |
|
| 312 | + return; |
|
| 313 | + } |
|
| 314 | + |
|
| 315 | + $chunkSize = $input->getOption('chunk-size'); |
|
| 316 | + |
|
| 317 | + $query = $fromDB->getQueryBuilder(); |
|
| 318 | + $query->automaticTablePrefix(false); |
|
| 319 | + $query->select($query->func()->count('*', 'num_entries')) |
|
| 320 | + ->from($table->getName()); |
|
| 321 | + $result = $query->execute(); |
|
| 322 | + $count = $result->fetchOne(); |
|
| 323 | + $result->closeCursor(); |
|
| 324 | + |
|
| 325 | + $numChunks = ceil($count / $chunkSize); |
|
| 326 | + if ($numChunks > 1) { |
|
| 327 | + $output->writeln('chunked query, ' . $numChunks . ' chunks'); |
|
| 328 | + } |
|
| 329 | + |
|
| 330 | + $progress = new ProgressBar($output, $count); |
|
| 331 | + $progress->setFormat('very_verbose'); |
|
| 332 | + $progress->start(); |
|
| 333 | + $redraw = $count > $chunkSize ? 100 : ($count > 100 ? 5 : 1); |
|
| 334 | + $progress->setRedrawFrequency($redraw); |
|
| 335 | + |
|
| 336 | + $query = $fromDB->getQueryBuilder(); |
|
| 337 | + $query->automaticTablePrefix(false); |
|
| 338 | + $query->select('*') |
|
| 339 | + ->from($table->getName()) |
|
| 340 | + ->setMaxResults($chunkSize); |
|
| 341 | + |
|
| 342 | + try { |
|
| 343 | + $orderColumns = $table->getPrimaryKeyColumns(); |
|
| 344 | + } catch (Exception $e) { |
|
| 345 | + $orderColumns = []; |
|
| 346 | + foreach ($table->getColumns() as $column) { |
|
| 347 | + $orderColumns[] = $column->getName(); |
|
| 348 | + } |
|
| 349 | + } |
|
| 350 | + |
|
| 351 | + foreach ($orderColumns as $column) { |
|
| 352 | + $query->addOrderBy($column); |
|
| 353 | + } |
|
| 354 | + |
|
| 355 | + $insertQuery = $toDB->getQueryBuilder(); |
|
| 356 | + $insertQuery->automaticTablePrefix(false); |
|
| 357 | + $insertQuery->insert($table->getName()); |
|
| 358 | + $parametersCreated = false; |
|
| 359 | + |
|
| 360 | + for ($chunk = 0; $chunk < $numChunks; $chunk++) { |
|
| 361 | + $query->setFirstResult($chunk * $chunkSize); |
|
| 362 | + |
|
| 363 | + $result = $query->execute(); |
|
| 364 | + |
|
| 365 | + while ($row = $result->fetch()) { |
|
| 366 | + $progress->advance(); |
|
| 367 | + if (!$parametersCreated) { |
|
| 368 | + foreach ($row as $key => $value) { |
|
| 369 | + $insertQuery->setValue($key, $insertQuery->createParameter($key)); |
|
| 370 | + } |
|
| 371 | + $parametersCreated = true; |
|
| 372 | + } |
|
| 373 | + |
|
| 374 | + foreach ($row as $key => $value) { |
|
| 375 | + $type = $this->getColumnType($table, $key); |
|
| 376 | + if ($type !== false) { |
|
| 377 | + $insertQuery->setParameter($key, $value, $type); |
|
| 378 | + } else { |
|
| 379 | + $insertQuery->setParameter($key, $value); |
|
| 380 | + } |
|
| 381 | + } |
|
| 382 | + $insertQuery->execute(); |
|
| 383 | + } |
|
| 384 | + $result->closeCursor(); |
|
| 385 | + } |
|
| 386 | + $progress->finish(); |
|
| 387 | + $output->writeln(''); |
|
| 388 | + } |
|
| 389 | + |
|
| 390 | + protected function getColumnType(Table $table, $columnName) { |
|
| 391 | + $tableName = $table->getName(); |
|
| 392 | + if (isset($this->columnTypes[$tableName][$columnName])) { |
|
| 393 | + return $this->columnTypes[$tableName][$columnName]; |
|
| 394 | + } |
|
| 395 | + |
|
| 396 | + $type = $table->getColumn($columnName)->getType()->getName(); |
|
| 397 | + |
|
| 398 | + switch ($type) { |
|
| 399 | + case Types::BLOB: |
|
| 400 | + case Types::TEXT: |
|
| 401 | + $this->columnTypes[$tableName][$columnName] = IQueryBuilder::PARAM_LOB; |
|
| 402 | + break; |
|
| 403 | + case Types::BOOLEAN: |
|
| 404 | + $this->columnTypes[$tableName][$columnName] = IQueryBuilder::PARAM_BOOL; |
|
| 405 | + break; |
|
| 406 | + default: |
|
| 407 | + $this->columnTypes[$tableName][$columnName] = false; |
|
| 408 | + } |
|
| 409 | + |
|
| 410 | + return $this->columnTypes[$tableName][$columnName]; |
|
| 411 | + } |
|
| 412 | + |
|
| 413 | + protected function convertDB(Connection $fromDB, Connection $toDB, array $tables, InputInterface $input, OutputInterface $output) { |
|
| 414 | + $this->config->setSystemValue('maintenance', true); |
|
| 415 | + $schema = $fromDB->createSchema(); |
|
| 416 | + |
|
| 417 | + try { |
|
| 418 | + // copy table rows |
|
| 419 | + foreach ($tables as $table) { |
|
| 420 | + $output->writeln('<info> - '.$table.'</info>'); |
|
| 421 | + $this->copyTable($fromDB, $toDB, $schema->getTable($table), $input, $output); |
|
| 422 | + } |
|
| 423 | + if ($input->getArgument('type') === 'pgsql') { |
|
| 424 | + $tools = new \OC\DB\PgSqlTools($this->config); |
|
| 425 | + $tools->resynchronizeDatabaseSequences($toDB); |
|
| 426 | + } |
|
| 427 | + // save new database config |
|
| 428 | + $this->saveDBInfo($input); |
|
| 429 | + } catch (\Exception $e) { |
|
| 430 | + $this->config->setSystemValue('maintenance', false); |
|
| 431 | + throw $e; |
|
| 432 | + } |
|
| 433 | + $this->config->setSystemValue('maintenance', false); |
|
| 434 | + } |
|
| 435 | + |
|
| 436 | + protected function saveDBInfo(InputInterface $input) { |
|
| 437 | + $type = $input->getArgument('type'); |
|
| 438 | + $username = $input->getArgument('username'); |
|
| 439 | + $dbHost = $input->getArgument('hostname'); |
|
| 440 | + $dbName = $input->getArgument('database'); |
|
| 441 | + $password = $input->getOption('password'); |
|
| 442 | + if ($input->getOption('port')) { |
|
| 443 | + $dbHost .= ':'.$input->getOption('port'); |
|
| 444 | + } |
|
| 445 | + |
|
| 446 | + $this->config->setSystemValues([ |
|
| 447 | + 'dbtype' => $type, |
|
| 448 | + 'dbname' => $dbName, |
|
| 449 | + 'dbhost' => $dbHost, |
|
| 450 | + 'dbuser' => $username, |
|
| 451 | + 'dbpassword' => $password, |
|
| 452 | + ]); |
|
| 453 | + } |
|
| 454 | + |
|
| 455 | + /** |
|
| 456 | + * Return possible values for the named option |
|
| 457 | + * |
|
| 458 | + * @param string $optionName |
|
| 459 | + * @param CompletionContext $context |
|
| 460 | + * @return string[] |
|
| 461 | + */ |
|
| 462 | + public function completeOptionValues($optionName, CompletionContext $context) { |
|
| 463 | + return []; |
|
| 464 | + } |
|
| 465 | + |
|
| 466 | + /** |
|
| 467 | + * Return possible values for the named argument |
|
| 468 | + * |
|
| 469 | + * @param string $argumentName |
|
| 470 | + * @param CompletionContext $context |
|
| 471 | + * @return string[] |
|
| 472 | + */ |
|
| 473 | + public function completeArgumentValues($argumentName, CompletionContext $context) { |
|
| 474 | + if ($argumentName === 'type') { |
|
| 475 | + return ['mysql', 'oci', 'pgsql']; |
|
| 476 | + } |
|
| 477 | + return []; |
|
| 478 | + } |
|
| 479 | 479 | } |