Phpml\Tokenization\NGramTokenizer::tokenize()

Complexity

Conditions	4
Paths	4

Size

Total Lines

19

Duplication

Lines	0
Ratio	0 %

Importance

Changes

0

<?php
    public function tokenize(string $text): array
    {
        $words = parent::tokenize($text);

        $ngramsFactory = new NGrams();

        $ngram_dataset = [];
        foreach ($words as $word) {
            $length = strlen($word);

            for ($i = 1; $i <= $length; $i++) {
                foreach ($ngramsFactory->ngrams(str_split($word), $i) as $ngram) {
                    $ngram_dataset[] = implode('', $ngram);
                }
            }
        }

        return $ngram_dataset;
    }
}