NGramTokenizerTest::testTokenize() - Code Metrics - onoi/tesa - Measure and Improve Code Quality continuously with Scrutinizer

NGramTokenizerTest::testTokenize() A
last analyzed 2018-01-27 17:32 UTC

↳ Parent: NGramTokenizerTest

Complexity

Conditions	2
Paths	2

Size

Total Lines	19
Code Lines	10

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
dl	0
loc	19
rs	9.4285
c	0
b	0
f	0
cc	2
eloc	10
nc	2
nop	3

<?php

namespace Onoi\Tesa\Tests;

use Onoi\Tesa\Tokenizer\NGramTokenizer;

/**
 * @covers \Onoi\Tesa\Tokenizer\NGramTokenizer
 * @group onoi-tesa
 *
 * @license GNU GPL v2+
 * @since 0.1
 *
 * @author mwjames
 */
class NGramTokenizerTest extends \PHPUnit_Framework_TestCase {

	public function testCanConstruct() {

		$this->assertInstanceOf(
			'\Onoi\Tesa\Tokenizer\NGramTokenizer',
			new NGramTokenizer()
		);
	}

	/**
	 * @dataProvider stringProvider
	 */
	public function testTokenize( $string, $ngram, $expected ) {

		if ( version_compare( phpversion(), '5.4', '<' ) ) {
			$this->markTestSkipped(
				"Ehh, PHP 5.3 returns with unexpected results"
			);
		}

		$instance = new NGramTokenizer( null, $ngram );

		$this->assertEquals(
			$expected,
			$instance->tokenize( $string )
		);

		$this->assertFalse(
			$instance->isWordTokenizer()
		);
	}

	public function testTokenizeWithStartEndMarker() {

		// http://cloudmark.github.io/Language-Detection
		$string = 'TEXT';

		$expected = array(
			'_tex',
			'text',
			'ext_',
			'xt__',
			't___'
		);

		$instance = new NGramTokenizer( null, 4 );
		$instance->withMarker( true );

		$this->assertEquals(
			$expected,
			$instance->tokenize( $string )
		);
	}

	public function testTokenizeWithStartEndMarker2() {

		$string = '教授は';

		$expected = array(
			'_教授',
			'教授は',
			'授は_',
			'は__'
		);

		$instance = new NGramTokenizer( null, 3 );
		$instance->withMarker( true );

		$this->assertEquals(
			$expected,
			$instance->tokenize( $string )
		);
	}
	public function testTokenizeWithOption() {

		$string = '红色中华';

		$tokenizer = $this->getMockBuilder( '\Onoi\Tesa\Tokenizer\Tokenizer' )
			->disableOriginalConstructor()
			->getMockForAbstractClass();

		$tokenizer->expects( $this->once() )
			->method( 'setOption' );

		$tokenizer->expects( $this->once() )
			->method( 'tokenize' )
			->with( $this->equalTo( $string ) )
			->will( $this->returnValue( array( $string ) ) );

		$instance = new NGramTokenizer( $tokenizer );

		$instance->setOption(
			NGramTokenizer::REGEX_EXEMPTION,
			array( 'Foo' )
		);

		$this->assertEquals(
			array( '红色', '色中', '中华' ),
			$instance->tokenize( $string )
		);
	}

	public function stringProvider() {

		$provider[] = array(
foreach ($collection as $item) {
    $myArray['foo'] = $item->getFoo();

    if ($item->hasBar()) {
        $myArray['bar'] = $item->getBar();
    }

    // do something with $myArray
}
			'TEXT',
			'4',
			array(
				'text'
			)
		);

		$provider[] = array(
			'12345678',
			'2',
			array(
				'12',
				'23',
				'34',
				'45',
				'56',
				'67',
				'78'
			)
		);

		$provider[] = array(
			'12345678',
			'3',
			array(
				'123',
				'234',
				'345',
				'456',
				'567',
				'678'
			)
		);

		$provider[] = array(
			'hello',
			'3',
			array(
				'hel',
				'ell',
				'llo'
			)
		);

		$provider[] = array(
			'Hello World!',
			'3',
			array(
				'hel',
				'ell',
				'llo',
				'lo ',
				'o w',
				' wo',
				'wor',
				'orl',
				'rld',
				'ld!'
			)
		);

		$provider[] = array(
			'Новости',
			'3',
			array(
				'нов',
				'ово',
				'вос',
				'ост',
				'сти'
			)
		);

		$provider[] = array(
			'1時36分更新',
			'3',
			array(
				'1時3',
				'時36',
				'36分',
				'6分更',
				'分更新'
			)
		);

		$provider[] = array(
			'こんにちは世界！',
			'2',
			array(
				'こん',
				'んに',
				'にち',
				'ちは',
				'は世',
				'世界',
				'界！'
			)
		);

		return $provider;
	}

}


1			<?php
2
3			namespace Onoi\Tesa\Tests;
4
5			use Onoi\Tesa\Tokenizer\NGramTokenizer;
6
7			/**
8			* @covers \Onoi\Tesa\Tokenizer\NGramTokenizer
9			* @group onoi-tesa
10			*
11			* @license GNU GPL v2+
12			* @since 0.1
13			*
14			* @author mwjames
15			*/
16			class NGramTokenizerTest extends \PHPUnit_Framework_TestCase {
17
18			public function testCanConstruct() {
19
20			$this->assertInstanceOf(
21			'\Onoi\Tesa\Tokenizer\NGramTokenizer',
22			new NGramTokenizer()
23			);
24			}
25
26			/**
27			* @dataProvider stringProvider
28			*/
29			public function testTokenize( $string, $ngram, $expected ) {
30
31			if ( version_compare( phpversion(), '5.4', '<' ) ) {
32			$this->markTestSkipped(
33			"Ehh, PHP 5.3 returns with unexpected results"
34			);
35			}
36
37			$instance = new NGramTokenizer( null, $ngram );
38
39			$this->assertEquals(
40			$expected,
41			$instance->tokenize( $string )
42			);
43
44			$this->assertFalse(
45			$instance->isWordTokenizer()
46			);
47			}
48
49			public function testTokenizeWithStartEndMarker() {
50
51			// http://cloudmark.github.io/Language-Detection
52			$string = 'TEXT';
53
54			$expected = array(
55			'_tex',
56			'text',
57			'ext_',
58			'xt__',
59			't___'
60			);
61
62			$instance = new NGramTokenizer( null, 4 );
63			$instance->withMarker( true );
64
65			$this->assertEquals(
66			$expected,
67			$instance->tokenize( $string )
68			);
69			}
70
71			public function testTokenizeWithStartEndMarker2() {
72
73			$string = '教授は';
74
75			$expected = array(
76			'_教授',
77			'教授は',
78			'授は_',
79			'は__'
80			);
81
82			$instance = new NGramTokenizer( null, 3 );
83			$instance->withMarker( true );
84
85			$this->assertEquals(
86			$expected,
87			$instance->tokenize( $string )
88			);
89			}
90			public function testTokenizeWithOption() {
91
92			$string = '红色中华';
93
94			$tokenizer = $this->getMockBuilder( '\Onoi\Tesa\Tokenizer\Tokenizer' )
95			->disableOriginalConstructor()
96			->getMockForAbstractClass();
97
98			$tokenizer->expects( $this->once() )
99			->method( 'setOption' );
100
101			$tokenizer->expects( $this->once() )
102			->method( 'tokenize' )
103			->with( $this->equalTo( $string ) )
104			->will( $this->returnValue( array( $string ) ) );
105
106			$instance = new NGramTokenizer( $tokenizer );
107
108			$instance->setOption(
109			NGramTokenizer::REGEX_EXEMPTION,
110			array( 'Foo' )
111			);
112
113			$this->assertEquals(
114			array( '红色', '色中', '中华' ),
115			$instance->tokenize( $string )
116			);
117			}
118
119			public function stringProvider() {
120
121			$provider[] = array(
			0 ignored issues – show Coding Style Comprehensibility introduced 2016-08-02 14:35 UTC by Report Bug Copy Issue Report `$provider` was never initialized. Although not strictly required by PHP, it is generally a good practice to add `$provider = array();` before regardless. Adding an explicit array definition is generally preferable to implicit array definition as it guarantees a stable state of the code. Let’s take a look at an example: foreach ($collection as $item) { $myArray['foo'] = $item->getFoo(); if ($item->hasBar()) { $myArray['bar'] = $item->getBar(); } // do something with $myArray } As you can see in this example, the array `$myArray` is initialized the first time when the foreach loop is entered. You can also see that the value of the `bar` key is only written conditionally; thus, its value might result from a previous iteration. This might or might not be intended. To make your intention clear, your code more readible and to avoid accidental bugs, we recommend to add an explicit initialization `$myArray = array()` either outside or inside the foreach loop. Loading history...
122			'TEXT',
123			'4',
124			array(
125			'text'
126			)
127			);
128
129			$provider[] = array(
130			'12345678',
131			'2',
132			array(
133			'12',
134			'23',
135			'34',
136			'45',
137			'56',
138			'67',
139			'78'
140			)
141			);
142
143			$provider[] = array(
144			'12345678',
145			'3',
146			array(
147			'123',
148			'234',
149			'345',
150			'456',
151			'567',
152			'678'
153			)
154			);
155
156			$provider[] = array(
157			'hello',
158			'3',
159			array(
160			'hel',
161			'ell',
162			'llo'
163			)
164			);
165
166			$provider[] = array(
167			'Hello World!',
168			'3',
169			array(
170			'hel',
171			'ell',
172			'llo',
173			'lo ',
174			'o w',
175			' wo',
176			'wor',
177			'orl',
178			'rld',
179			'ld!'
180			)
181			);
182
183			$provider[] = array(
184			'Новости',
185			'3',
186			array(
187			'нов',
188			'ово',
189			'вос',
190			'ост',
191			'сти'
192			)
193			);
194
195			$provider[] = array(
196			'1時36分更新',
197			'3',
198			array(
199			'1時3',
200			'時36',
201			'36分',
202			'6分更',
203			'分更新'
204			)
205			);
206
207			$provider[] = array(
208			'こんにちは世界！',
209			'2',
210			array(
211			'こん',
212			'んに',
213			'にち',
214			'ちは',
215			'は世',
216			'世界',
217			'界！'
218			)
219			);
220
221			return $provider;
222			}
223
224			}
225

onoi / tesa

NGramTokenizerTest::testTokenize() A last analyzed 2018-01-27 17:32 UTC

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like

NGramTokenizerTest::testTokenize() A
last analyzed 2018-01-27 17:32 UTC