Phpml\Tokenization\NGramTokenizer

Complexity

Total Complexity

5

Size/Duplication

Total Lines	42
Duplicated Lines	0 %

Coupling/Cohesion

Components	0
Dependencies	2

Importance

Changes

0

1 Method

Rating	Name	Duplication	Size	Complexity
A	tokenize()	0	36	5

<?php

declare(strict_types=1);

namespace Phpml\Tokenization;

/**
 * Class NGramTokenizer
 */
class NGramTokenizer extends WordTokenizer
{
    /**
     * {@inheritdoc}
     */
    public function tokenize(string $text): array
    {
        $words = parent::tokenize($text);

        $length = 1;
        foreach ($words as $word) {
            $candidate_length = strlen($word);
            if ($candidate_length > $length) {
                $length = $candidate_length;
            }
        }

        $ngramsFactory = new \drupol\phpngrams\NGrams();

        $ngram_dataset = [];
        foreach ($words as $word) {
            $length = strlen($word);

            for ($i = 1; $i <= $length; $i++) {
                $ngram_dataset = array_merge(
                    $ngram_dataset,
                    array_map(
                        'implode',
                        iterator_to_array(
                            $ngramsFactory->ngrams(
                                str_split($word),
                                $i
                            )
                        )
                    )
                );
            }
        }

        return $ngram_dataset;
    }
}