Phpml\Tokenization\NGramTokenizer

Complexity

Total Complexity

4

Size/Duplication

Total Lines	25
Duplicated Lines	0 %

Coupling/Cohesion

Components	0
Dependencies	2

Importance

Changes

0

1 Method

Rating	Name	Duplication	Size	Complexity
A	tokenize()	0	19	4

<?php

declare(strict_types=1);

namespace Phpml\Tokenization;

use drupol\phpngrams\NGrams;

/**
 * Class NGramTokenizer
 */
class NGramTokenizer extends WordTokenizer
{
    /**
     * {@inheritdoc}
     */
    public function tokenize(string $text): array
    {
        $words = parent::tokenize($text);

        $ngramsFactory = new NGrams();

        $ngram_dataset = [];
        foreach ($words as $word) {
            $length = strlen($word);

            for ($i = 1; $i <= $length; $i++) {
                foreach ($ngramsFactory->ngrams(str_split($word), $i) as $ngram) {
                    $ngram_dataset[] = implode('', $ngram);
                }
            }
        }

        return $ngram_dataset;
    }
}