PunctuationRegExTokenizer::tokenize() - Code Metrics - onoi/tesa - Measure and Improve Code Quality continuously with Scrutinizer

PunctuationRegExTokenizer::tokenize() A
last analyzed 2018-01-27 17:32 UTC

↳ Parent: PunctuationRegExTokenizer

Complexity

Conditions	3
Paths	4

Size

Total Lines	20
Code Lines	11

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	11
CRAP Score	3.0327

Importance

Changes

Metric	Value
dl	0
loc	20
ccs	11
cts	13
cp	0.8462
rs	9.4285
c	0
b	0
f	0
cc	3
eloc	11
nc	4
nop	1
crap	3.0327

<?php

namespace Onoi\Tesa\Tokenizer;

/**
 * @license GNU GPL v2+
 * @since 0.1
 *
 * @author mwjames
 */
class PunctuationRegExTokenizer implements Tokenizer {

	/**
	 * @var Tokenizer
	 */
	private $tokenizer;

	/**
	 * @var string
	 */
	private $patternExemption = '';

	/**
	 * @since 0.1
	 *
	 * @param Tokenizer|null $tokenizer
	 */
	public function __construct( Tokenizer $tokenizer = null ) {
		$this->tokenizer = $tokenizer;
	}

	/**
	 * @since 0.1
	 *
	 * {@inheritDoc}
	 */
	public function setOption( $name, $value ) {

		if ( $this->tokenizer !== null ) {
			$this->tokenizer->setOption( $name, $value );
		}

		if ( $name === self::REGEX_EXEMPTION ) {
			$this->patternExemption = $value;
		}
	}

	/**
	 * @since 0.1
	 *
	 * {@inheritDoc}
	 */
	public function isWordTokenizer() {
		return $this->tokenizer !== null ? $this->tokenizer->isWordTokenizer() : true;
	}

	/**
	 * @since 0.1
	 *
	 * @param string $string
	 *
	 * @return array|false
	 */
	public function tokenize( $string ) {

		if ( $this->tokenizer !== null ) {
			$string = implode( " ", $this->tokenizer->tokenize( $string ) );
		}

		$pattern = str_replace(
			$this->patternExemption,
			'',
			'＿－・，、；：！？．。…◆★◇□■（）【】《》〈〉；：“”＂〃＇｀［］｛｝｢｣＠＊＼／＆＃％｀＾＋＜＝＞｜～≪≫─＄＂_\-･,､;:!?.｡()[\]{}「」@*\/&#%`^+<=>|~«»$"\s'
		);

		$result = preg_split( '/[' . $pattern . ']+/u', $string, null, PREG_SPLIT_NO_EMPTY );

		if ( $result === false ) {
			$result = array();
		}

		return $result;
	}

}


1		<?php
2
3		namespace Onoi\Tesa\Tokenizer;
4
5		/**
6		* @license GNU GPL v2+
7		* @since 0.1
8		*
9		* @author mwjames
10		*/
11		class PunctuationRegExTokenizer implements Tokenizer {
12
13		/**
14		* @var Tokenizer
15		*/
16		private $tokenizer;
17
18		/**
19		* @var string
20		*/
21		private $patternExemption = '';
22
23		/**
24		* @since 0.1
25		*
26		* @param Tokenizer\|null $tokenizer
27		*/
28	7	public function __construct( Tokenizer $tokenizer = null ) {
29	7	$this->tokenizer = $tokenizer;
30	7	}
31
32		/**
33		* @since 0.1
34		*
35		* {@inheritDoc}
36		*/
37	4	public function setOption( $name, $value ) {
38
39	4	if ( $this->tokenizer !== null ) {
40	1	$this->tokenizer->setOption( $name, $value );
41	1	}
42
43	4	if ( $name === self::REGEX_EXEMPTION ) {
44	4	$this->patternExemption = $value;
45	4	}
46	4	}
47
48		/**
49		* @since 0.1
50		*
51		* {@inheritDoc}
52		*/
53	4	public function isWordTokenizer() {
54	4	return $this->tokenizer !== null ? $this->tokenizer->isWordTokenizer() : true;
55		}
56
57		/**
58		* @since 0.1
59		*
60		* @param string $string
61		*
62		* @return array\|false
63		*/
64	5	public function tokenize( $string ) {
65
66	5	if ( $this->tokenizer !== null ) {
67	1	$string = implode( " ", $this->tokenizer->tokenize( $string ) );
68	1	}
69
70	5	$pattern = str_replace(
71	5	$this->patternExemption,
72	5	'',
73		'＿－・，、；：！？．。…◆★◇□■（）【】《》〈〉；：“”＂〃＇｀［］｛｝｢｣＠＊＼／＆＃％｀＾＋＜＝＞｜～≪≫─＄＂_\-･,､;:!?.｡()[\]{}「」@*\/&#%`^+<=>\|~«»$"\s'
74	5	);
75
76	5	$result = preg_split( '/[' . $pattern . ']+/u', $string, null, PREG_SPLIT_NO_EMPTY );
77
78	5	if ( $result === false ) {
79		$result = array();
80		}
81
82	5	return $result;
83		}
84
85		}
86

onoi / tesa

PunctuationRegExTokenizer::tokenize() A last analyzed 2018-01-27 17:32 UTC

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like

PunctuationRegExTokenizer::tokenize() A
last analyzed 2018-01-27 17:32 UTC