Phpml\Tokenization\NGramTokenizer::tokenize()

Complexity

Conditions	5
Paths	7

Size

Total Lines

20

Duplication

Lines	0
Ratio	0 %

Importance

Changes

0

<?php
    public function tokenize(string $text): array
    {
        $ngram_dataset = [];

        foreach (parent::tokenize($text) as $word) {
            $lengths = range(1, strlen($word), 1);

            if ($this->sizes !== null) {
                $lengths = $this->sizes;
            }

            foreach ($lengths as $length) {
                foreach ($this->ngramsFactory->ngrams(str_split($word), $length) as $ngram) {
                    $ngram_dataset[] = implode('', $ngram);
                }
            }
        }

        return $ngram_dataset;
    }
}