Phpml\Tokenization\WhitespaceTokenizer

Complexity

Total Complexity

1

Size/Duplication

Total Lines	12
Duplicated Lines	0 %

Coupling/Cohesion

Components	0
Dependencies	0

1 Method

Rating	Name	Duplication	Size	Complexity
A	tokenize()	0	4	1

<?php

declare (strict_types = 1);

namespace Phpml\Tokenization;

class WhitespaceTokenizer implements Tokenizer
{
    /**
     * @param string $text
     *
     * @return array
     */
    public function tokenize(string $text): array
    {
        return preg_split('/[\pZ\pC]+/u', $text, -1, PREG_SPLIT_NO_EMPTY);
    }
}