Phpml\Tokenization\NGramTokenizer::tokenize()

Complexity

Conditions	5
Paths	9

Size

Total Lines

36

Duplication

Lines	0
Ratio	0 %

Importance

Changes

0

<?php
    public function tokenize(string $text): array
    {
        $words = parent::tokenize($text);

        $length = 1;
        foreach ($words as $word) {
            $candidate_length = strlen($word);
            if ($candidate_length > $length) {
                $length = $candidate_length;
            }
        }

        $ngramsFactory = new \drupol\phpngrams\NGrams();

        $ngram_dataset = [];
        foreach ($words as $word) {
            $length = strlen($word);

            for ($i = 1; $i <= $length; $i++) {
                $ngram_dataset = array_merge(
                    $ngram_dataset,
                    array_map(
                        'implode',
                        iterator_to_array(
                            $ngramsFactory->ngrams(
                                str_split($word),
                                $i
                            )
                        )
                    )
                );
            }
        }

        return $ngram_dataset;
    }
}