PunctuationRegExTokenizer::tokenize() - Code Metrics - Inspection of "Add SanitizerFactory" - onoi/tesa - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( 9401a4...2cc7a2 )

by mw

created 2016-08-02 14:22 UTC

PunctuationRegExTokenizer::tokenize() A

↳ Parent: PunctuationRegExTokenizer

Complexity

Conditions	3
Paths	4

Size

Total Lines	20
Code Lines	11

Duplication

Lines	0
Ratio	0 %

Importance

Changes	1
Bugs	0	Features	1

Metric	Value
c	1
b	0
f	1
dl	0
loc	20
rs	9.4285
cc	3
eloc	11
nc	4
nop	1

<?php

namespace Onoi\Tesa\Tokenizer;

/**
 * @license GNU GPL v2+
 * @since 0.1
 *
 * @author mwjames
 */
class PunctuationRegExTokenizer implements Tokenizer {

	/**
	 * @var Tokenizer
	 */
	private $tokenizer;

	/**
	 * @var string
	 */
	private $patternExemption = '';

	/**
	 * @since 0.1
	 *
	 * @param Tokenizer|null $tokenizer
	 */
	public function __construct( Tokenizer $tokenizer = null ) {
		$this->tokenizer = $tokenizer;
	}

	/**
	 * @since 0.1
	 *
	 * {@inheritDoc}
	 */
	public function setOption( $name, $value ) {

		if ( $this->tokenizer !== null ) {
			$this->tokenizer->setOption( $name, $value );
		}

		if ( $name === self::REGEX_EXEMPTION ) {
			$this->patternExemption = $value;
		}
	}

	/**
	 * @since 0.1
	 *
	 * {@inheritDoc}
	 */
	public function isWordTokenizer() {
		return $this->tokenizer !== null ? $this->tokenizer->isWordTokenizer() : true;
	}

	/**
	 * @since 0.1
	 *
	 * @param string $string
	 *
	 * @return array|false
	 */
	public function tokenize( $string ) {

		if ( $this->tokenizer !== null ) {
			$string = implode( " ", $this->tokenizer->tokenize( $string ) );
		}

		$pattern = str_replace(
			$this->patternExemption,
			'',
			'＿－・，、；：！？．。…◆★◇□■（）【】《》〈〉；：“”＂〃＇｀［］｛｝｢｣＠＊＼／＆＃％｀＾＋＜＝＞｜～≪≫─＄＂_\-･,､;:!?.｡()[\]{}「」@*\/&#%`^+<=>|~«»$"\s'
		);

		$result = preg_split( '/[' . $pattern . ']+/u', $string, null, PREG_SPLIT_NO_EMPTY );

		if ( $result === false ) {
			$result = array();
		}

		return $result;
	}

}


1			<?php
2
3			namespace Onoi\Tesa\Tokenizer;
4
5			/**
6			* @license GNU GPL v2+
7			* @since 0.1
8			*
9			* @author mwjames
10			*/
11			class PunctuationRegExTokenizer implements Tokenizer {
12
13			/**
14			* @var Tokenizer
15			*/
16			private $tokenizer;
17
18			/**
19			* @var string
20			*/
21			private $patternExemption = '';
22
23			/**
24			* @since 0.1
25			*
26			* @param Tokenizer\|null $tokenizer
27			*/
28			public function __construct( Tokenizer $tokenizer = null ) {
29			$this->tokenizer = $tokenizer;
30			}
31
32			/**
33			* @since 0.1
34			*
35			* {@inheritDoc}
36			*/
37			public function setOption( $name, $value ) {
38
39			if ( $this->tokenizer !== null ) {
40			$this->tokenizer->setOption( $name, $value );
41			}
42
43			if ( $name === self::REGEX_EXEMPTION ) {
44			$this->patternExemption = $value;
45			}
46			}
47
48			/**
49			* @since 0.1
50			*
51			* {@inheritDoc}
52			*/
53			public function isWordTokenizer() {
54			return $this->tokenizer !== null ? $this->tokenizer->isWordTokenizer() : true;
55			}
56
57			/**
58			* @since 0.1
59			*
60			* @param string $string
61			*
62			* @return array\|false
63			*/
64			public function tokenize( $string ) {
65
66			if ( $this->tokenizer !== null ) {
67			$string = implode( " ", $this->tokenizer->tokenize( $string ) );
68			}
69
70			$pattern = str_replace(
71			$this->patternExemption,
72			'',
73			'＿－・，、；：！？．。…◆★◇□■（）【】《》〈〉；：“”＂〃＇｀［］｛｝｢｣＠＊＼／＆＃％｀＾＋＜＝＞｜～≪≫─＄＂_\-･,､;:!?.｡()[\]{}「」@*\/&#%`^+<=>\|~«»$"\s'
74			);
75
76			$result = preg_split( '/[' . $pattern . ']+/u', $string, null, PREG_SPLIT_NO_EMPTY );
77
78			if ( $result === false ) {
79			$result = array();
80			}
81
82			return $result;
83			}
84
85			}
86

onoi / tesa

Push — master ( 9401a4...2cc7a2 )

PunctuationRegExTokenizer::tokenize() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like