Phpml\Tokenization\NGramTokenizer

Complexity

Total Complexity

6

Size/Duplication

Total Lines	49
Duplicated Lines	0 %

Coupling/Cohesion

Components	1
Dependencies	3

Importance

Changes

0

2 Methods

Rating	Name	Duplication	Size	Complexity
A	__construct()	0	5	1
A	tokenize()	0	20	5

<?php

declare(strict_types=1);

namespace Phpml\Tokenization;

use drupol\phpngrams\NGrams;

/**
 * Class NGramTokenizer
 */
class NGramTokenizer extends WordTokenizer
{
    /**
     * The sizes.
     *
     * @var array|null
     */
    private $sizes;

    /**
     * The NGram factory.
     *
     * @var \drupol\phpngrams\NGramsInterface
     */
    private $ngramsFactory;

    /**
     * NGramTokenizer constructor.
     */
    public function __construct(?array $sizes = null)
    {
        $this->sizes = $sizes;
        $this->ngramsFactory = new NGrams();
    }

    /**
     * {@inheritdoc}
     */
    public function tokenize(string $text): array
    {
        $ngram_dataset = [];

        foreach (parent::tokenize($text) as $word) {
            $lengths = range(1, strlen($word), 1);

            if ($this->sizes !== null) {
                $lengths = $this->sizes;
            }

            foreach ($lengths as $length) {
                foreach ($this->ngramsFactory->ngrams(str_split($word), $length) as $ngram) {
                    $ngram_dataset[] = implode('', $ngram);
                }
            }
        }

        return $ngram_dataset;
    }
}