tburry /
pquery
Complex classes like TokenizerBase often do a lot of different things. To break such a class down, we need to identify a cohesive component within that class. A common approach to find such a component is to look for fields/methods that share the same prefixes, or suffixes. You can also have a look at the cohesion graph to spot any un-connected, or weakly-connected components.
Once you have determined the fields that belong together, you can apply the Extract Class refactoring. If the component makes sense as a sub-class, Extract Subclass is also a candidate, and is often faster.
While breaking up the class, it is a good idea to analyze how other classes use TokenizerBase, and based on these observations, apply Extract Interface, too.
| 1 | <?php |
||
| 32 | class TokenizerBase {
|
||
| 33 | |||
| 34 | /** |
||
| 35 | * NULL Token, used at end of document (parsing should stop after this token) |
||
| 36 | */ |
||
| 37 | const TOK_NULL = 0; |
||
| 38 | /** |
||
| 39 | * Unknown token, used at unidentified character |
||
| 40 | */ |
||
| 41 | const TOK_UNKNOWN = 1; |
||
| 42 | /** |
||
| 43 | * Whitespace token, used with whitespace |
||
| 44 | */ |
||
| 45 | const TOK_WHITESPACE = 2; |
||
| 46 | /** |
||
| 47 | * Identifier token, used with identifiers |
||
| 48 | */ |
||
| 49 | const TOK_IDENTIFIER = 3; |
||
| 50 | |||
| 51 | /** |
||
| 52 | * The document that is being tokenized |
||
| 53 | * @var string |
||
| 54 | * @internal Public for faster access! |
||
| 55 | * @see setDoc() |
||
| 56 | * @see getDoc() |
||
| 57 | * @access private |
||
| 58 | */ |
||
| 59 | var $doc = ''; |
||
|
|
|||
| 60 | |||
| 61 | /** |
||
| 62 | * The size of the document (length of string) |
||
| 63 | * @var int |
||
| 64 | * @internal Public for faster access! |
||
| 65 | * @see $doc |
||
| 66 | * @access private |
||
| 67 | */ |
||
| 68 | var $size = 0; |
||
| 69 | |||
| 70 | /** |
||
| 71 | * Current (character) position in the document |
||
| 72 | * @var int |
||
| 73 | * @internal Public for faster access! |
||
| 74 | * @see setPos() |
||
| 75 | * @see getPos() |
||
| 76 | * @access private |
||
| 77 | */ |
||
| 78 | var $pos = 0; |
||
| 79 | |||
| 80 | /** |
||
| 81 | * Current (Line/Column) position in document |
||
| 82 | * @var array (Current_Line, Line_Starting_Pos) |
||
| 83 | * @internal Public for faster access! |
||
| 84 | * @see getLinePos() |
||
| 85 | * @access private |
||
| 86 | */ |
||
| 87 | var $line_pos = array(0, 0); |
||
| 88 | |||
| 89 | /** |
||
| 90 | * Current token |
||
| 91 | * @var int |
||
| 92 | * @internal Public for faster access! |
||
| 93 | * @see getToken() |
||
| 94 | * @access private |
||
| 95 | */ |
||
| 96 | var $token = self::TOK_NULL; |
||
| 97 | |||
| 98 | /** |
||
| 99 | * Start position of token. If NULL, then current position is used. |
||
| 100 | * @var int |
||
| 101 | * @internal Public for faster access! |
||
| 102 | * @see getTokenString() |
||
| 103 | * @access private |
||
| 104 | */ |
||
| 105 | var $token_start = null; |
||
| 106 | |||
| 107 | /** |
||
| 108 | * List with all the character that can be considered as whitespace |
||
| 109 | * @var array|string |
||
| 110 | * @internal Variable is public + associated array for faster access! |
||
| 111 | * @internal array(' ' => true) will recognize space (' ') as whitespace
|
||
| 112 | * @internal String will be converted to array in constructor |
||
| 113 | * @internal Result token will be {@link self::TOK_WHITESPACE};
|
||
| 114 | * @see setWhitespace() |
||
| 115 | * @see getWhitespace() |
||
| 116 | * @access private |
||
| 117 | */ |
||
| 118 | var $whitespace = " \t\n\r\0\x0B"; |
||
| 119 | |||
| 120 | /** |
||
| 121 | * List with all the character that can be considered as identifier |
||
| 122 | * @var array|string |
||
| 123 | * @internal Variable is public + associated array for faster access! |
||
| 124 | * @internal array('a' => true) will recognize 'a' as identifier
|
||
| 125 | * @internal String will be converted to array in constructor |
||
| 126 | * @internal Result token will be {@link self::TOK_IDENTIFIER};
|
||
| 127 | * @see setIdentifiers() |
||
| 128 | * @see getIdentifiers() |
||
| 129 | * @access private |
||
| 130 | */ |
||
| 131 | var $identifiers = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890_'; |
||
| 132 | |||
| 133 | /** |
||
| 134 | * All characters that should be mapped to a token/function that cannot be considered as whitespace or identifier |
||
| 135 | * @var array |
||
| 136 | * @internal Variable is public + associated array for faster access! |
||
| 137 | * @internal array('a' => 'parse_a') will call $this->parse_a() if it matches the character 'a'
|
||
| 138 | * @internal array('a' => self::TOK_A) will set token to TOK_A if it matches the character 'a'
|
||
| 139 | * @see mapChar() |
||
| 140 | * @see unmapChar() |
||
| 141 | * @access private |
||
| 142 | */ |
||
| 143 | var $custom_char_map = array(); |
||
| 144 | |||
| 145 | /** |
||
| 146 | * Automatically built character map. Built using {@link $identifiers}, {@link $whitespace} and {@link $custom_char_map}
|
||
| 147 | * @var array |
||
| 148 | * @internal Public for faster access! |
||
| 149 | * @access private |
||
| 150 | */ |
||
| 151 | var $char_map = array(); |
||
| 152 | |||
| 153 | /** |
||
| 154 | * All errors found while parsing the document |
||
| 155 | * @var array |
||
| 156 | * @see addError() |
||
| 157 | */ |
||
| 158 | var $errors = array(); |
||
| 159 | |||
| 160 | /** |
||
| 161 | * Class constructor |
||
| 162 | * @param string $doc Document to be tokenized |
||
| 163 | * @param int $pos Position to start parsing |
||
| 164 | * @see setDoc() |
||
| 165 | * @see setPos() |
||
| 166 | */ |
||
| 167 | 37 | function __construct($doc = '', $pos = 0) {
|
|
| 173 | |||
| 174 | #php4 PHP4 class constructor compatibility |
||
| 175 | #function TokenizerBase($doc = '', $pos = 0) {return $this->__construct($doc, $pos);}
|
||
| 176 | #php4e |
||
| 177 | |||
| 178 | /** |
||
| 179 | * Sets target document |
||
| 180 | * @param string $doc Document to be tokenized |
||
| 181 | * @param int $pos Position to start parsing |
||
| 182 | * @see getDoc() |
||
| 183 | * @see setPos() |
||
| 184 | */ |
||
| 185 | 37 | function setDoc($doc, $pos = 0) {
|
|
| 190 | |||
| 191 | /** |
||
| 192 | * Returns target document |
||
| 193 | * @return string |
||
| 194 | * @see setDoc() |
||
| 195 | */ |
||
| 196 | function getDoc() {
|
||
| 199 | |||
| 200 | /** |
||
| 201 | * Sets position in document |
||
| 202 | * @param int $pos |
||
| 203 | * @see getPos() |
||
| 204 | */ |
||
| 205 | 37 | function setPos($pos = 0) {
|
|
| 210 | |||
| 211 | /** |
||
| 212 | * Returns current position in document (Index) |
||
| 213 | * @return int |
||
| 214 | * @see setPos() |
||
| 215 | */ |
||
| 216 | 1 | function getPos() {
|
|
| 219 | |||
| 220 | /** |
||
| 221 | * Returns current position in document (Line/Char) |
||
| 222 | * @return array array(Line, Column) |
||
| 223 | */ |
||
| 224 | function getLinePos() {
|
||
| 227 | |||
| 228 | /** |
||
| 229 | * Returns current token |
||
| 230 | * @return int |
||
| 231 | * @see $token |
||
| 232 | */ |
||
| 233 | function getToken() {
|
||
| 236 | |||
| 237 | /** |
||
| 238 | * Returns current token as string |
||
| 239 | * @param int $start_offset Offset from token start |
||
| 240 | * @param int $end_offset Offset from token end |
||
| 241 | * @return string |
||
| 242 | */ |
||
| 243 | 37 | function getTokenString($start_offset = 0, $end_offset = 0) {
|
|
| 248 | |||
| 249 | /** |
||
| 250 | * Sets characters to be recognized as whitespace |
||
| 251 | * |
||
| 252 | * Used like: setWhitespace('ab') or setWhitespace(array('a' => true, 'b', 'c'));
|
||
| 253 | * @param string|array $ws |
||
| 254 | * @see getWhitespace(); |
||
| 255 | */ |
||
| 256 | 37 | function setWhitespace($ws) {
|
|
| 264 | |||
| 265 | /** |
||
| 266 | * Returns whitespace characters as string/array |
||
| 267 | * @param bool $as_string Should the result be a string or an array? |
||
| 268 | * @return string|array |
||
| 269 | * @see setWhitespace() |
||
| 270 | */ |
||
| 271 | function getWhitespace($as_string = true) {
|
||
| 275 | |||
| 276 | /** |
||
| 277 | * Sets characters to be recognized as identifier |
||
| 278 | * |
||
| 279 | * Used like: setIdentifiers('ab') or setIdentifiers(array('a' => true, 'b', 'c'));
|
||
| 280 | * @param string|array $ident |
||
| 281 | * @see getIdentifiers(); |
||
| 282 | */ |
||
| 283 | 37 | function setIdentifiers($ident) {
|
|
| 291 | |||
| 292 | /** |
||
| 293 | * Returns identifier characters as string/array |
||
| 294 | * @param bool $as_string Should the result be a string or an array? |
||
| 295 | * @return string|array |
||
| 296 | * @see setIdentifiers() |
||
| 297 | */ |
||
| 298 | function getIdentifiers($as_string = true) {
|
||
| 302 | |||
| 303 | /** |
||
| 304 | * Maps a custom character to a token/function |
||
| 305 | * |
||
| 306 | * Used like: mapChar('a', self::{@link TOK_IDENTIFIER}) or mapChar('a', 'parse_identifier');
|
||
| 307 | * @param string $char Character that should be mapped. If set, it will be overridden |
||
| 308 | * @param int|string $map If function name, then $this->function will be called, otherwise token is set to $map |
||
| 309 | * @see unmapChar() |
||
| 310 | */ |
||
| 311 | function mapChar($char, $map) {
|
||
| 315 | |||
| 316 | /** |
||
| 317 | * Removes a char mapped with {@link mapChar()}
|
||
| 318 | * @param string $char Character that should be unmapped |
||
| 319 | * @see mapChar() |
||
| 320 | */ |
||
| 321 | function unmapChar($char) {
|
||
| 325 | |||
| 326 | /** |
||
| 327 | * Builds the {@link $map_char} array
|
||
| 328 | * @internal Builds single array that maps all characters. Gets called if {@link $whitespace}, {@link $identifiers} or {@link $custom_char_map} get modified
|
||
| 329 | */ |
||
| 330 | 37 | protected function buildCharMap() {
|
|
| 343 | |||
| 344 | /** |
||
| 345 | * Add error to the array and appends current position |
||
| 346 | * @param string $error |
||
| 347 | */ |
||
| 348 | function addError($error) {
|
||
| 351 | |||
| 352 | /** |
||
| 353 | * Parse line breaks and increase line number |
||
| 354 | * @internal Gets called to process line breaks |
||
| 355 | */ |
||
| 356 | 34 | protected function parse_linebreak() {
|
|
| 368 | |||
| 369 | /** |
||
| 370 | * Parse whitespace |
||
| 371 | * @return int Token |
||
| 372 | * @internal Gets called with {@link $whitespace} characters
|
||
| 373 | */ |
||
| 374 | 4 | protected function parse_whitespace() {
|
|
| 388 | |||
| 389 | /** |
||
| 390 | * Parse identifiers |
||
| 391 | * @return int Token |
||
| 392 | * @internal Gets called with {@link $identifiers} characters
|
||
| 393 | */ |
||
| 394 | 37 | protected function parse_identifier() {
|
|
| 402 | |||
| 403 | /** |
||
| 404 | * Continues to the next token |
||
| 405 | * @return int Next token ({@link TOK_NULL} if none)
|
||
| 406 | */ |
||
| 407 | 37 | function next() {
|
|
| 424 | |||
| 425 | /** |
||
| 426 | * Finds the next token, but skips whitespace |
||
| 427 | * @return int Next token ({@link TOK_NULL} if none)
|
||
| 428 | */ |
||
| 429 | 37 | function next_no_whitespace() {
|
|
| 450 | |||
| 451 | /** |
||
| 452 | * Finds the next token using stop characters. |
||
| 453 | * |
||
| 454 | * Used like: next_search('abc') or next_search(array('a' => true, 'b' => true, 'c' => true));
|
||
| 455 | * @param string|array $characters Characters to search for |
||
| 456 | * @param bool $callback Should the function check the charmap after finding a character? |
||
| 457 | * @return int Next token ({@link TOK_NULL} if none)
|
||
| 458 | */ |
||
| 459 | 9 | function next_search($characters, $callback = true) {
|
|
| 483 | |||
| 484 | /** |
||
| 485 | * Finds the next token by searching for a string |
||
| 486 | * @param string $needle The needle that's being searched for |
||
| 487 | * @param bool $callback Should the function check the charmap after finding the needle? |
||
| 488 | * @return int Next token ({@link TOK_NULL} if none)
|
||
| 489 | */ |
||
| 490 | 37 | function next_pos($needle, $callback = true) {
|
|
| 525 | |||
| 526 | /** |
||
| 527 | * Expect a specific token or character. Adds error if token doesn't match. |
||
| 528 | * @param string|int $token Character or token to expect |
||
| 529 | * @param bool|int $do_next Go to next character before evaluating. 1 for next char, true to ignore whitespace |
||
| 530 | * @param bool|int $try_next Try next character if current doesn't match. 1 for next char, true to ignore whitespace |
||
| 531 | * @param bool|int $next_on_match Go to next character after evaluating. 1 for next char, true to ignore whitespace |
||
| 532 | * @return bool |
||
| 533 | */ |
||
| 534 | protected function expect($token, $do_next = true, $try_next = false, $next_on_match = 1) {
|
||
| 564 | } |
||
| 565 | |||
| 566 | ?> |
||
Loading history...
The PSR-2 coding standard requires that all properties in a class have their visibility explicitly declared. If you declare a property using
the property is implicitly global.
To learn more about the PSR-2, please see the PHP-FIG site on the PSR-2.