Hal\Component\Token\Tokenizer::tokenize()

Complexity

Conditions	3
Paths	3

Size

Total Lines	22
Code Lines	13

Duplication

Lines	0
Ratio	0 %

Importance

Changes	1
Bugs	1	Features	0

<?php
    public function tokenize($filename) {
        //
        // fixes memory problems with large files
        // https://github.com/Halleck45/PhpMetrics/issues/13
        $size = filesize($filename);
        $limit = 102400; // around 100 Ko
        if($size > $limit) {
            $tokens = array();
            $hwnd = fopen($filename, 'r');
            while (!feof($hwnd)) {
                $content = stream_get_line($hwnd, $limit);
                // string is arbitrary splitted, so content can be incorrect
                // for example: "Unterminated comment starting..."
                $content .= '/* */';
                $tokens = array_merge($tokens, token_get_all($this->cleanup($content)));
                unset($content);
            }
            return new TokenCollection($tokens);
        }

        return new TokenCollection(token_get_all($this->cleanup(file_get_contents($filename))));
    }

}