wikimedia /
mediawiki
Duplicate code is one of the most pungent code smells. A rule that is often used is to re-structure code once it is duplicated in three or more places.
Common duplication problems, and corresponding solutions are:
Complex classes like TextPassDumper often do a lot of different things. To break such a class down, we need to identify a cohesive component within that class. A common approach to find such a component is to look for fields/methods that share the same prefixes, or suffixes. You can also have a look at the cohesion graph to spot any un-connected, or weakly-connected components.
Once you have determined the fields that belong together, you can apply the Extract Class refactoring. If the component makes sense as a sub-class, Extract Subclass is also a candidate, and is often faster.
While breaking up the class, it is a good idea to analyze how other classes use TextPassDumper, and based on these observations, apply Extract Interface, too.
| 1 | <?php |
||
| 33 | class TextPassDumper extends BackupDumper { |
||
| 34 | public $prefetch = null; |
||
| 35 | |||
| 36 | // when we spend more than maxTimeAllowed seconds on this run, we continue |
||
| 37 | // processing until we write out the next complete page, then save output file(s), |
||
| 38 | // rename it/them and open new one(s) |
||
| 39 | public $maxTimeAllowed = 0; // 0 = no limit |
||
| 40 | |||
| 41 | protected $input = "php://stdin"; |
||
| 42 | protected $history = WikiExporter::FULL; |
||
| 43 | protected $fetchCount = 0; |
||
| 44 | protected $prefetchCount = 0; |
||
| 45 | protected $prefetchCountLast = 0; |
||
| 46 | protected $fetchCountLast = 0; |
||
| 47 | |||
| 48 | protected $maxFailures = 5; |
||
| 49 | protected $maxConsecutiveFailedTextRetrievals = 200; |
||
| 50 | protected $failureTimeout = 5; // Seconds to sleep after db failure |
||
| 51 | |||
| 52 | protected $bufferSize = 524288; // In bytes. Maximum size to read from the stub in on go. |
||
| 53 | |||
| 54 | protected $php = "php"; |
||
| 55 | protected $spawn = false; |
||
| 56 | |||
| 57 | /** |
||
| 58 | * @var bool|resource |
||
| 59 | */ |
||
| 60 | protected $spawnProc = false; |
||
| 61 | |||
| 62 | /** |
||
| 63 | * @var bool|resource |
||
| 64 | */ |
||
| 65 | protected $spawnWrite = false; |
||
| 66 | |||
| 67 | /** |
||
| 68 | * @var bool|resource |
||
| 69 | */ |
||
| 70 | protected $spawnRead = false; |
||
| 71 | |||
| 72 | /** |
||
| 73 | * @var bool|resource |
||
| 74 | */ |
||
| 75 | protected $spawnErr = false; |
||
| 76 | |||
| 77 | protected $xmlwriterobj = false; |
||
| 78 | |||
| 79 | protected $timeExceeded = false; |
||
| 80 | protected $firstPageWritten = false; |
||
| 81 | protected $lastPageWritten = false; |
||
| 82 | protected $checkpointJustWritten = false; |
||
| 83 | protected $checkpointFiles = []; |
||
| 84 | |||
| 85 | /** |
||
| 86 | * @var DatabaseBase |
||
| 87 | */ |
||
| 88 | protected $db; |
||
| 89 | |||
| 90 | /** |
||
| 91 | * @param array $args For backward compatibility |
||
| 92 | */ |
||
| 93 | function __construct( $args = null ) { |
||
| 128 | |||
| 129 | function execute() { |
||
| 133 | |||
| 134 | function processOptions() { |
||
| 177 | |||
| 178 | /** |
||
| 179 | * Drop the database connection $this->db and try to get a new one. |
||
| 180 | * |
||
| 181 | * This function tries to get a /different/ connection if this is |
||
| 182 | * possible. Hence, (if this is possible) it switches to a different |
||
| 183 | * failover upon each call. |
||
| 184 | * |
||
| 185 | * This function resets $this->lb and closes all connections on it. |
||
| 186 | * |
||
| 187 | * @throws MWException |
||
| 188 | */ |
||
| 189 | function rotateDb() { |
||
| 228 | |||
| 229 | function initProgress( $history = WikiExporter::FULL ) { |
||
| 233 | |||
| 234 | function dump( $history, $text = WikiExporter::TEXT ) { |
||
| 274 | |||
| 275 | function processFileOpt( $opt ) { |
||
| 306 | |||
| 307 | /** |
||
| 308 | * Overridden to include prefetch ratio if enabled. |
||
| 309 | */ |
||
| 310 | function showReport() { |
||
| 369 | |||
| 370 | function setTimeExceeded() { |
||
| 373 | |||
| 374 | function checkIfTimeExceeded() { |
||
| 383 | |||
| 384 | function finalOptionCheck() { |
||
| 407 | |||
| 408 | /** |
||
| 409 | * @throws MWException Failure to parse XML input |
||
| 410 | * @param string $input |
||
| 411 | * @return bool |
||
| 412 | */ |
||
| 413 | function readDump( $input ) { |
||
| 414 | $this->buffer = ""; |
||
| 415 | $this->openElement = false; |
||
| 416 | $this->atStart = true; |
||
| 417 | $this->state = ""; |
||
| 418 | $this->lastName = ""; |
||
| 419 | $this->thisPage = 0; |
||
| 420 | $this->thisRev = 0; |
||
| 421 | $this->thisRevModel = null; |
||
| 422 | $this->thisRevFormat = null; |
||
| 423 | |||
| 424 | $parser = xml_parser_create( "UTF-8" ); |
||
| 425 | xml_parser_set_option( $parser, XML_OPTION_CASE_FOLDING, false ); |
||
| 426 | |||
| 427 | xml_set_element_handler( |
||
| 428 | $parser, |
||
| 429 | [ $this, 'startElement' ], |
||
| 430 | [ $this, 'endElement' ] |
||
| 431 | ); |
||
| 432 | xml_set_character_data_handler( $parser, [ $this, 'characterData' ] ); |
||
| 433 | |||
| 434 | $offset = 0; // for context extraction on error reporting |
||
| 435 | do { |
||
| 436 | if ( $this->checkIfTimeExceeded() ) { |
||
| 437 | $this->setTimeExceeded(); |
||
| 438 | } |
||
| 439 | $chunk = fread( $input, $this->bufferSize ); |
||
| 440 | if ( !xml_parse( $parser, $chunk, feof( $input ) ) ) { |
||
| 441 | wfDebug( "TextDumpPass::readDump encountered XML parsing error\n" ); |
||
| 442 | |||
| 443 | $byte = xml_get_current_byte_index( $parser ); |
||
| 444 | $msg = wfMessage( 'xml-error-string', |
||
| 445 | 'XML import parse failure', |
||
| 446 | xml_get_current_line_number( $parser ), |
||
| 447 | xml_get_current_column_number( $parser ), |
||
| 448 | $byte . ( is_null( $chunk ) ? null : ( '; "' . substr( $chunk, $byte - $offset, 16 ) . '"' ) ), |
||
| 449 | xml_error_string( xml_get_error_code( $parser ) ) )->escaped(); |
||
| 450 | |||
| 451 | xml_parser_free( $parser ); |
||
| 452 | |||
| 453 | throw new MWException( $msg ); |
||
| 454 | } |
||
| 455 | $offset += strlen( $chunk ); |
||
| 456 | } while ( $chunk !== false && !feof( $input ) ); |
||
| 457 | if ( $this->maxTimeAllowed ) { |
||
| 458 | $filenameList = (array)$this->egress->getFilenames(); |
||
| 459 | // we wrote some stuff after last checkpoint that needs renamed |
||
| 460 | if ( file_exists( $filenameList[0] ) ) { |
||
| 461 | $newFilenames = []; |
||
| 462 | # we might have just written the header and footer and had no |
||
| 463 | # pages or revisions written... perhaps they were all deleted |
||
| 464 | # there's no pageID 0 so we use that. the caller is responsible |
||
| 465 | # for deciding what to do with a file containing only the |
||
| 466 | # siteinfo information and the mw tags. |
||
| 467 | if ( !$this->firstPageWritten ) { |
||
| 468 | $firstPageID = str_pad( 0, 9, "0", STR_PAD_LEFT ); |
||
| 469 | $lastPageID = str_pad( 0, 9, "0", STR_PAD_LEFT ); |
||
| 470 | } else { |
||
| 471 | $firstPageID = str_pad( $this->firstPageWritten, 9, "0", STR_PAD_LEFT ); |
||
| 472 | $lastPageID = str_pad( $this->lastPageWritten, 9, "0", STR_PAD_LEFT ); |
||
| 473 | } |
||
| 474 | |||
| 475 | $filenameCount = count( $filenameList ); |
||
| 476 | View Code Duplication | for ( $i = 0; $i < $filenameCount; $i++ ) { |
|
| 477 | $checkpointNameFilledIn = sprintf( $this->checkpointFiles[$i], $firstPageID, $lastPageID ); |
||
| 478 | $fileinfo = pathinfo( $filenameList[$i] ); |
||
| 479 | $newFilenames[] = $fileinfo['dirname'] . '/' . $checkpointNameFilledIn; |
||
| 480 | } |
||
| 481 | $this->egress->closeAndRename( $newFilenames ); |
||
| 482 | } |
||
| 483 | } |
||
| 484 | xml_parser_free( $parser ); |
||
| 485 | |||
| 486 | return true; |
||
| 487 | } |
||
| 488 | |||
| 489 | /** |
||
| 490 | * Applies applicable export transformations to $text. |
||
| 491 | * |
||
| 492 | * @param string $text |
||
| 493 | * @param string $model |
||
| 494 | * @param string|null $format |
||
| 495 | * |
||
| 496 | * @return string |
||
| 497 | */ |
||
| 498 | private function exportTransform( $text, $model, $format = null ) { |
||
| 512 | |||
| 513 | /** |
||
| 514 | * Tries to get the revision text for a revision id. |
||
| 515 | * Export transformations are applied if the content model can is given or can be |
||
| 516 | * determined from the database. |
||
| 517 | * |
||
| 518 | * Upon errors, retries (Up to $this->maxFailures tries each call). |
||
| 519 | * If still no good revision get could be found even after this retrying, "" is returned. |
||
| 520 | * If no good revision text could be returned for |
||
| 521 | * $this->maxConsecutiveFailedTextRetrievals consecutive calls to getText, MWException |
||
| 522 | * is thrown. |
||
| 523 | * |
||
| 524 | * @param string $id The revision id to get the text for |
||
| 525 | * @param string|bool|null $model The content model used to determine |
||
| 526 | * applicable export transformations. |
||
| 527 | * If $model is null, it will be determined from the database. |
||
| 528 | * @param string|null $format The content format used when applying export transformations. |
||
| 529 | * |
||
| 530 | * @throws MWException |
||
| 531 | * @return string The revision text for $id, or "" |
||
| 532 | */ |
||
| 533 | function getText( $id, $model = null, $format = null ) { |
||
| 534 | global $wgContentHandlerUseDB; |
||
| 535 | |||
| 536 | $prefetchNotTried = true; // Whether or not we already tried to get the text via prefetch. |
||
| 537 | $text = false; // The candidate for a good text. false if no proper value. |
||
| 538 | $failures = 0; // The number of times, this invocation of getText already failed. |
||
| 539 | |||
| 540 | // The number of times getText failed without yielding a good text in between. |
||
| 541 | static $consecutiveFailedTextRetrievals = 0; |
||
| 542 | |||
| 543 | $this->fetchCount++; |
||
| 544 | |||
| 545 | // To allow to simply return on success and do not have to worry about book keeping, |
||
| 546 | // we assume, this fetch works (possible after some retries). Nevertheless, we koop |
||
| 547 | // the old value, so we can restore it, if problems occur (See after the while loop). |
||
| 548 | $oldConsecutiveFailedTextRetrievals = $consecutiveFailedTextRetrievals; |
||
| 549 | $consecutiveFailedTextRetrievals = 0; |
||
| 550 | |||
| 551 | if ( $model === null && $wgContentHandlerUseDB ) { |
||
| 552 | $row = $this->db->selectRow( |
||
| 553 | 'revision', |
||
| 554 | [ 'rev_content_model', 'rev_content_format' ], |
||
| 555 | [ 'rev_id' => $this->thisRev ], |
||
| 556 | __METHOD__ |
||
| 557 | ); |
||
| 558 | |||
| 559 | if ( $row ) { |
||
| 560 | $model = $row->rev_content_model; |
||
| 561 | $format = $row->rev_content_format; |
||
| 562 | } |
||
| 563 | } |
||
| 564 | |||
| 565 | if ( $model === null || $model === '' ) { |
||
| 566 | $model = false; |
||
| 567 | } |
||
| 568 | |||
| 569 | while ( $failures < $this->maxFailures ) { |
||
| 570 | |||
| 571 | // As soon as we found a good text for the $id, we will return immediately. |
||
| 572 | // Hence, if we make it past the try catch block, we know that we did not |
||
| 573 | // find a good text. |
||
| 574 | |||
| 575 | try { |
||
| 576 | // Step 1: Get some text (or reuse from previous iteratuon if checking |
||
| 577 | // for plausibility failed) |
||
| 578 | |||
| 579 | // Trying to get prefetch, if it has not been tried before |
||
| 580 | if ( $text === false && isset( $this->prefetch ) && $prefetchNotTried ) { |
||
| 581 | $prefetchNotTried = false; |
||
| 582 | $tryIsPrefetch = true; |
||
| 583 | $text = $this->prefetch->prefetch( intval( $this->thisPage ), |
||
| 584 | intval( $this->thisRev ) ); |
||
| 585 | |||
| 586 | if ( $text === null ) { |
||
| 587 | $text = false; |
||
| 588 | } |
||
| 589 | |||
| 590 | View Code Duplication | if ( is_string( $text ) && $model !== false ) { |
|
| 591 | // Apply export transformation to text coming from an old dump. |
||
| 592 | // The purpose of this transformation is to convert up from legacy |
||
| 593 | // formats, which may still be used in the older dump that is used |
||
| 594 | // for pre-fetching. Applying the transformation again should not |
||
| 595 | // interfere with content that is already in the correct form. |
||
| 596 | $text = $this->exportTransform( $text, $model, $format ); |
||
| 597 | } |
||
| 598 | } |
||
| 599 | |||
| 600 | if ( $text === false ) { |
||
| 601 | // Fallback to asking the database |
||
| 602 | $tryIsPrefetch = false; |
||
| 603 | if ( $this->spawn ) { |
||
| 604 | $text = $this->getTextSpawned( $id ); |
||
| 605 | } else { |
||
| 606 | $text = $this->getTextDb( $id ); |
||
| 607 | } |
||
| 608 | |||
| 609 | View Code Duplication | if ( $text !== false && $model !== false ) { |
|
| 610 | // Apply export transformation to text coming from the database. |
||
| 611 | // Prefetched text should already have transformations applied. |
||
| 612 | $text = $this->exportTransform( $text, $model, $format ); |
||
| 613 | } |
||
| 614 | |||
| 615 | // No more checks for texts from DB for now. |
||
| 616 | // If we received something that is not false, |
||
| 617 | // We treat it as good text, regardless of whether it actually is or is not |
||
| 618 | if ( $text !== false ) { |
||
| 619 | return $text; |
||
| 620 | } |
||
| 621 | } |
||
| 622 | |||
| 623 | if ( $text === false ) { |
||
| 624 | throw new MWException( "Generic error while obtaining text for id " . $id ); |
||
| 625 | } |
||
| 626 | |||
| 627 | // We received a good candidate for the text of $id via some method |
||
| 628 | |||
| 629 | // Step 2: Checking for plausibility and return the text if it is |
||
| 630 | // plausible |
||
| 631 | $revID = intval( $this->thisRev ); |
||
| 632 | if ( !isset( $this->db ) ) { |
||
| 633 | throw new MWException( "No database available" ); |
||
| 634 | } |
||
| 635 | |||
| 636 | if ( $model !== CONTENT_MODEL_WIKITEXT ) { |
||
| 637 | $revLength = strlen( $text ); |
||
| 638 | } else { |
||
| 639 | $revLength = $this->db->selectField( 'revision', 'rev_len', [ 'rev_id' => $revID ] ); |
||
| 640 | } |
||
| 641 | |||
| 642 | if ( strlen( $text ) == $revLength ) { |
||
| 643 | if ( $tryIsPrefetch ) { |
||
| 644 | $this->prefetchCount++; |
||
| 645 | } |
||
| 646 | |||
| 647 | return $text; |
||
| 648 | } |
||
| 649 | |||
| 650 | $text = false; |
||
| 651 | throw new MWException( "Received text is unplausible for id " . $id ); |
||
| 652 | } catch ( Exception $e ) { |
||
| 653 | $msg = "getting/checking text " . $id . " failed (" . $e->getMessage() . ")"; |
||
| 654 | if ( $failures + 1 < $this->maxFailures ) { |
||
| 655 | $msg .= " (Will retry " . ( $this->maxFailures - $failures - 1 ) . " more times)"; |
||
| 656 | } |
||
| 657 | $this->progress( $msg ); |
||
| 658 | } |
||
| 659 | |||
| 660 | // Something went wrong; we did not a text that was plausible :( |
||
| 661 | $failures++; |
||
| 662 | |||
| 663 | // A failure in a prefetch hit does not warrant resetting db connection etc. |
||
| 664 | if ( !$tryIsPrefetch ) { |
||
| 665 | // After backing off for some time, we try to reboot the whole process as |
||
| 666 | // much as possible to not carry over failures from one part to the other |
||
| 667 | // parts |
||
| 668 | sleep( $this->failureTimeout ); |
||
| 669 | try { |
||
| 670 | $this->rotateDb(); |
||
| 671 | if ( $this->spawn ) { |
||
| 672 | $this->closeSpawn(); |
||
| 673 | $this->openSpawn(); |
||
| 674 | } |
||
| 675 | } catch ( Exception $e ) { |
||
| 676 | $this->progress( "Rebooting getText infrastructure failed (" . $e->getMessage() . ")" . |
||
| 677 | " Trying to continue anyways" ); |
||
| 678 | } |
||
| 679 | } |
||
| 680 | } |
||
| 681 | |||
| 682 | // Retirieving a good text for $id failed (at least) maxFailures times. |
||
| 683 | // We abort for this $id. |
||
| 684 | |||
| 685 | // Restoring the consecutive failures, and maybe aborting, if the dump |
||
| 686 | // is too broken. |
||
| 687 | $consecutiveFailedTextRetrievals = $oldConsecutiveFailedTextRetrievals + 1; |
||
| 688 | if ( $consecutiveFailedTextRetrievals > $this->maxConsecutiveFailedTextRetrievals ) { |
||
| 689 | throw new MWException( "Graceful storage failure" ); |
||
| 690 | } |
||
| 691 | |||
| 692 | return ""; |
||
| 693 | } |
||
| 694 | |||
| 695 | /** |
||
| 696 | * May throw a database error if, say, the server dies during query. |
||
| 697 | * @param int $id |
||
| 698 | * @return bool|string |
||
| 699 | * @throws MWException |
||
| 700 | */ |
||
| 701 | private function getTextDb( $id ) { |
||
| 702 | global $wgContLang; |
||
| 703 | if ( !isset( $this->db ) ) { |
||
| 704 | throw new MWException( __METHOD__ . "No database available" ); |
||
| 705 | } |
||
| 706 | $row = $this->db->selectRow( 'text', |
||
| 707 | [ 'old_text', 'old_flags' ], |
||
| 708 | [ 'old_id' => $id ], |
||
| 709 | __METHOD__ ); |
||
| 710 | $text = Revision::getRevisionText( $row ); |
||
| 711 | if ( $text === false ) { |
||
| 712 | return false; |
||
| 713 | } |
||
| 714 | $stripped = str_replace( "\r", "", $text ); |
||
| 715 | $normalized = $wgContLang->normalize( $stripped ); |
||
| 716 | |||
| 717 | return $normalized; |
||
| 718 | } |
||
| 719 | |||
| 720 | private function getTextSpawned( $id ) { |
||
| 731 | |||
| 732 | function openSpawn() { |
||
| 733 | global $IP; |
||
| 734 | |||
| 735 | if ( file_exists( "$IP/../multiversion/MWScript.php" ) ) { |
||
| 736 | $cmd = implode( " ", |
||
| 737 | array_map( 'wfEscapeShellArg', |
||
| 738 | [ |
||
| 739 | $this->php, |
||
| 740 | "$IP/../multiversion/MWScript.php", |
||
| 741 | "fetchText.php", |
||
| 742 | '--wiki', wfWikiID() ] ) ); |
||
| 743 | } else { |
||
| 744 | $cmd = implode( " ", |
||
| 745 | array_map( 'wfEscapeShellArg', |
||
| 746 | [ |
||
| 747 | $this->php, |
||
| 748 | "$IP/maintenance/fetchText.php", |
||
| 749 | '--wiki', wfWikiID() ] ) ); |
||
| 750 | } |
||
| 751 | $spec = [ |
||
| 752 | 0 => [ "pipe", "r" ], |
||
| 753 | 1 => [ "pipe", "w" ], |
||
| 754 | 2 => [ "file", "/dev/null", "a" ] ]; |
||
| 755 | $pipes = []; |
||
| 756 | |||
| 757 | $this->progress( "Spawning database subprocess: $cmd" ); |
||
| 758 | $this->spawnProc = proc_open( $cmd, $spec, $pipes ); |
||
| 759 | if ( !$this->spawnProc ) { |
||
| 760 | $this->progress( "Subprocess spawn failed." ); |
||
| 761 | |||
| 762 | return false; |
||
| 763 | } |
||
| 764 | list( |
||
| 765 | $this->spawnWrite, // -> stdin |
||
| 766 | $this->spawnRead, // <- stdout |
||
| 767 | ) = $pipes; |
||
| 768 | |||
| 769 | return true; |
||
| 770 | } |
||
| 771 | |||
| 772 | private function closeSpawn() { |
||
| 792 | |||
| 793 | private function getTextSpawnedOnce( $id ) { |
||
| 854 | |||
| 855 | function startElement( $parser, $name, $attribs ) { |
||
| 856 | $this->checkpointJustWritten = false; |
||
| 857 | |||
| 858 | $this->clearOpenElement( null ); |
||
| 859 | $this->lastName = $name; |
||
| 860 | |||
| 861 | if ( $name == 'revision' ) { |
||
| 862 | $this->state = $name; |
||
| 863 | $this->egress->writeOpenPage( null, $this->buffer ); |
||
| 864 | $this->buffer = ""; |
||
| 865 | } elseif ( $name == 'page' ) { |
||
| 866 | $this->state = $name; |
||
| 867 | if ( $this->atStart ) { |
||
| 868 | $this->egress->writeOpenStream( $this->buffer ); |
||
| 869 | $this->buffer = ""; |
||
| 870 | $this->atStart = false; |
||
| 871 | } |
||
| 872 | } |
||
| 873 | |||
| 874 | if ( $name == "text" && isset( $attribs['id'] ) ) { |
||
| 875 | $id = $attribs['id']; |
||
| 876 | $model = trim( $this->thisRevModel ); |
||
| 877 | $format = trim( $this->thisRevFormat ); |
||
| 878 | |||
| 879 | $model = $model === '' ? null : $model; |
||
| 880 | $format = $format === '' ? null : $format; |
||
| 881 | |||
| 882 | $text = $this->getText( $id, $model, $format ); |
||
| 883 | $this->openElement = [ $name, [ 'xml:space' => 'preserve' ] ]; |
||
| 884 | if ( strlen( $text ) > 0 ) { |
||
| 885 | $this->characterData( $parser, $text ); |
||
| 886 | } |
||
| 887 | } else { |
||
| 888 | $this->openElement = [ $name, $attribs ]; |
||
| 889 | } |
||
| 890 | } |
||
| 891 | |||
| 892 | function endElement( $parser, $name ) { |
||
| 951 | |||
| 952 | function characterData( $parser, $data ) { |
||
| 976 | |||
| 977 | function clearOpenElement( $style ) { |
||
| 983 | } |
||
| 984 | |||
| 985 | $maintClass = 'TextPassDumper'; |
||
| 986 | require_once RUN_MAINTENANCE_IF_MAIN; |
||
| 987 |
Loading history...
The PSR-1: Basic Coding Standard recommends that a file should either introduce new symbols, that is classes, functions, constants or similar, or have side effects. Side effects are anything that executes logic, like for example printing output, changing ini settings or writing to a file.
The idea behind this recommendation is that merely auto-loading a class should not change the state of an application. It also promotes a cleaner style of programming and makes your code less prone to errors, because the logic is not spread out all over the place.
To learn more about the PSR-1, please see the PHP-FIG site on the PSR-1.