NGramTokenizer::__construct() - Code Metrics - Inspection of "Issue #349: Provide a new NGramTokenizer." - php-ai/php-ml - Measure and Improve Code Quality continuously with Scrutinizer

Test Setup Failed

Pull Request — master (#350)

by Pol

created 2019-02-06 10:16 UTC

NGramTokenizer::__construct() A

↳ Parent: NGramTokenizer

Complexity

Conditions	1
Paths	1

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
dl	0
loc	5
rs	10
c	0
b	0
f	0
cc	1
nc	1
nop	1

<?php

declare(strict_types=1);

namespace Phpml\Tokenization;

use drupol\phpngrams\NGrams;

/**
 * Class NGramTokenizer
 */
class NGramTokenizer extends WordTokenizer
{
    /**
     * The sizes.
     *
     * @var array|null
     */
    private $sizes;

    /**
     * The NGram factory.
     *
     * @var \drupol\phpngrams\NGramsInterface
     */
    private $ngramsFactory;

    /**
     * NGramTokenizer constructor.
     */
    public function __construct(?array $sizes = null)
    {
        $this->sizes = $sizes;
        $this->ngramsFactory = new NGrams();
    }

    /**
     * {@inheritdoc}
     */
    public function tokenize(string $text): array
    {
        $ngram_dataset = [];

        foreach (parent::tokenize($text) as $word) {
            $lengths = range(1, strlen($word), 1);

            if ($this->sizes !== null) {
                $lengths = $this->sizes;
            }

            foreach ($lengths as $length) {
                foreach ($this->ngramsFactory->ngrams(str_split($word), $length) as $ngram) {
                    $ngram_dataset[] = implode('', $ngram);
                }
            }
        }

        return $ngram_dataset;
    }
}


1			<?php
2
3			declare(strict_types=1);
4
5			namespace Phpml\Tokenization;
6
7			use drupol\phpngrams\NGrams;
8
9			/**
10			* Class NGramTokenizer
11			*/
12			class NGramTokenizer extends WordTokenizer
13			{
14			/**
15			* The sizes.
16			*
17			* @var array\|null
18			*/
19			private $sizes;
20
21			/**
22			* The NGram factory.
23			*
24			* @var \drupol\phpngrams\NGramsInterface
25			*/
26			private $ngramsFactory;
27
28			/**
29			* NGramTokenizer constructor.
30			*/
31			public function __construct(?array $sizes = null)
32			{
33			$this->sizes = $sizes;
34			$this->ngramsFactory = new NGrams();
35			}
36
37			/**
38			* {@inheritdoc}
39			*/
40			public function tokenize(string $text): array
41			{
42			$ngram_dataset = [];
43
44			foreach (parent::tokenize($text) as $word) {
45			$lengths = range(1, strlen($word), 1);
46
47			if ($this->sizes !== null) {
48			$lengths = $this->sizes;
49			}
50
51			foreach ($lengths as $length) {
52			foreach ($this->ngramsFactory->ngrams(str_split($word), $length) as $ngram) {
53			$ngram_dataset[] = implode('', $ngram);
54			}
55			}
56			}
57
58			return $ngram_dataset;
59			}
60			}
61

php-ai / php-ml

Pull Request — master (#350)

NGramTokenizer::__construct() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like