vanderlee / php-sentence

src/Sentence.php

Spacing +2 added lines, -2 removed lines

@@ -121,7 +121,7 @@ 
         $merges = [];
         $merge = '';
 
-        $filtered = array_filter($punctuations, function ($p) {
+        $filtered = array_filter($punctuations, function($p) {
             return $p !== '';
         });
 
@@ -366,7 +366,7 @@ 
      */
     private static function trimSentences($sentences)
     {
-        return array_map(function ($sentence) {
+        return array_map(function($sentence) {
             return Multibyte::trim($sentence);
         }, $sentences);
     }

Indentation +409 added lines, -409 removed lines

@@ -17,415 +17,415 @@
 class Sentence
 {
 
-    /**
-     * Specify this flag with the split method to trim whitespace.
-     */
-    const SPLIT_TRIM = 0x1;
-
-    /**
-     * List of characters used to terminate sentences.
-     *
-     * @var string[]
-     */
-    private $terminals = ['.', '!', '?'];
-
-    /**
-     * List of characters used for abbreviations.
-     *
-     * @var string[]
-     */
-    private $abbreviators = ['.'];
-
-    /**
-     * List of float numbers in the text
-     *
-     * @var string[]
-     */
-    private $floatNumbers = [];
-
-    /**
-     * Clean floating point numbers by replace them with their md5 hash
-     *
-     * @param string $text
-     *
-     * @return string
-     */
-    private function floatNumberClean(string $text)
-    {
-        preg_match_all('!\d+(?:\.\d+)?!', $text, $matches);
-
-        foreach ($matches[0] as $floatNumber) {
-            $this->floatNumbers[$floatNumber] = md5($floatNumber);
-
-            $text = str_replace($floatNumber, md5($floatNumber), $text);
-        }
-
-        return $text;
-    }
-
-    /**
-     * Revert the hashed floating number back
-     *
-     * @param string[] $text
-     *
-     * @return string[]
-     */
-    private function floatNumberRevert($text)
-    {
+	/**
+	 * Specify this flag with the split method to trim whitespace.
+	 */
+	const SPLIT_TRIM = 0x1;
+
+	/**
+	 * List of characters used to terminate sentences.
+	 *
+	 * @var string[]
+	 */
+	private $terminals = ['.', '!', '?'];
+
+	/**
+	 * List of characters used for abbreviations.
+	 *
+	 * @var string[]
+	 */
+	private $abbreviators = ['.'];
+
+	/**
+	 * List of float numbers in the text
+	 *
+	 * @var string[]
+	 */
+	private $floatNumbers = [];
+
+	/**
+	 * Clean floating point numbers by replace them with their md5 hash
+	 *
+	 * @param string $text
+	 *
+	 * @return string
+	 */
+	private function floatNumberClean(string $text)
+	{
+		preg_match_all('!\d+(?:\.\d+)?!', $text, $matches);
+
+		foreach ($matches[0] as $floatNumber) {
+			$this->floatNumbers[$floatNumber] = md5($floatNumber);
+
+			$text = str_replace($floatNumber, md5($floatNumber), $text);
+		}
+
+		return $text;
+	}
+
+	/**
+	 * Revert the hashed floating number back
+	 *
+	 * @param string[] $text
+	 *
+	 * @return string[]
+	 */
+	private function floatNumberRevert($text)
+	{
         
-        return array_map(function($value) {
-            foreach ($this->floatNumbers as $number => $hash) {
-                $value = str_replace($hash, $number, $value);
-            }
-            return $value;
-        }, $text);
-    }
-
-    /**
-     * Breaks a piece of text into lines by linebreak.
-     * Eats up any linebreak characters as if one.
-     *
-     * Multibyte.php safe
-     *
-     * @param string $text
-     * @return string[]
-     */
-    private static function linebreakSplit($text)
-    {
-        $lines = [];
-        $line = '';
-
-        foreach (Multibyte::split('([\r\n]+)', $text, -1, PREG_SPLIT_DELIM_CAPTURE) as $part) {
-            $line .= $part;
-            if (Multibyte::trim($part) === '') {
-                $lines[] = $line;
-                $line = '';
-            }
-        }
-        $lines[] = $line;
-
-        return $lines;
-    }
-
-    /**
-     * Splits an array of lines by (consecutive sequences of)
-     * terminals, keeping terminals.
-     *
-     * Multibyte.php safe (atleast for UTF-8)
-     *
-     * For example:
-     *    "There ... is. More!"
-     *        ... becomes ...
-     *    [ "There ", "...", " is", ".", " More", "!" ]
-     *
-     * @param string $line
-     * @return string[]
-     */
-    private function punctuationSplit($line)
-    {
-        $parts = [];
-
-        $chars = preg_split('//u', $line, -1, PREG_SPLIT_NO_EMPTY); // This is UTF8 multibyte safe!
-        $is_terminal = in_array($chars[0], $this->terminals);
-
-        $part = '';
-        foreach ($chars as $index => $char) {
-            if (in_array($char, $this->terminals) !== $is_terminal) {
-                $parts[] = $part;
-                $part = '';
-                $is_terminal = !$is_terminal;
-            }
-            $part .= $char;
-        }
-
-        if (!empty($part)) {
-            $parts[] = $part;
-        }
-
-        return $parts;
-    }
-
-    /**
-     * Appends each terminal item after it's preceding
-     * non-terminals.
-     *
-     * Multibyte.php safe (atleast for UTF-8)
-     *
-     * For example:
-     *    [ "There ", "...", " is", ".", " More", "!" ]
-     *        ... becomes ...
-     *    [ "There ... is.", "More!" ]
-     *
-     * @param string[] $punctuations
-     * @return string[]
-     */
-    private function punctuationMerge($punctuations)
-    {
-        $definite_terminals = array_diff($this->terminals, $this->abbreviators);
-
-        $merges = [];
-        $merge = '';
-
-        $filtered = array_filter($punctuations, function ($p) {
-            return $p !== '';
-        });
-
-        foreach ($filtered as $punctuation) {
-            $merge .= $punctuation;
-            if (mb_strlen($punctuation) === 1
-                && in_array($punctuation, $this->terminals)) {
-                $merges[] = $merge;
-                $merge = '';
-            } else {
-                foreach ($definite_terminals as $terminal) {
-                    if (mb_strpos($punctuation, $terminal) !== false) {
-                        $merges[] = $merge;
-                        $merge = '';
-                        break;
-                    }
-                }
-            }
-        }
-        if (!empty($merge)) {
-            $merges[] = $merge;
-        }
-
-        return $merges;
-    }
-
-    /**
-     * Looks for capitalized abbreviations & includes them with the following fragment.
-     *
-     * For example:
-     *    [ "Last week, former director of the F.B.I. James B. Comey was fired. Mr. Comey was not available for comment." ]
-     *        ... becomes ...
-     *    [ "Last week, former director of the F.B.I. James B. Comey was fired." ]
-     *  [ "Mr. Comey was not available for comment." ]
-     *
-     * @param string[] $fragments
-     * @return string[]
-     */
-    private function abbreviationMerge($fragments)
-    {
-        $return_fragment = [];
-
-        $previous_fragment = '';
-        $previous_is_abbreviation = false;
-        $i = 0;
-        foreach ($fragments as $fragment) {
-            $is_abbreviation = self::isAbreviation($fragment);
-
-            // merge previous fragment with this
-            if ($previous_is_abbreviation) {
-                $fragment = $previous_fragment . $fragment;
-            }
-            $return_fragment[$i] = $fragment;
-
-            $previous_is_abbreviation = $is_abbreviation;
-            $previous_fragment = $fragment;
-
-            // only increment if this isn't an abbreviation
-            if (!$is_abbreviation) {
-                $i++;
-            }
-        }
-        return $return_fragment;
-    }
-
-    /**
-     * Check if the last word of fragment starts with a Capital, ends in "." & has less than 3 characters.
-     *
-     * @param $fragment
-     * @return bool
-     */
-    private static function isAbreviation($fragment)
-    {
-        $words = mb_split('\s+', Multibyte::trim($fragment));
-
-        $word_count = count($words);
-
-        $last_word = Multibyte::trim($words[$word_count - 1]);
-        $last_is_capital = preg_match('#^\p{Lu}#u', $last_word);
-        $last_is_abbreviation = mb_substr(Multibyte::trim($fragment), -1) === '.';
-
-        return $last_is_capital > 0
-            && $last_is_abbreviation > 0
-            && mb_strlen($last_word) <= 3;
-    }
-
-    /**
-     * Merges any part starting with a closing parenthesis ')' to the previous
-     * part.
-     *
-     * @param string[] $parts
-     * @return string[]
-     */
-    private function parenthesesMerge($parts)
-    {
-        $subsentences = [];
-
-        foreach ($parts as $part) {
-            if ($part[0] === ')') {
-                $subsentences[count($subsentences) - 1] .= $part;
-            } else {
-                $subsentences[] = $part;
-            }
-        }
-
-        return $subsentences;
-    }
-
-    /**
-     * Looks for closing quotes to include them with the previous statement.
-     * "That was very interesting," he said.
-     * "That was very interesting."
-     *
-     * @param string[] $statements
-     * @return string[]
-     */
-    private function closeQuotesMerge($statements)
-    {
-        $i = 0;
-        $previous_statement = '';
-        $return = [];
-        foreach ($statements as $statement) {
-            if (self::isEndQuote($statement)) {
-                $statement = $previous_statement . $statement;
-            } else {
-                $i++;
-            }
-
-            $return[$i] = $statement;
-            $previous_statement = $statement;
-        }
-
-        return $return;
-    }
-
-    /**
-     * Check if the entire string is a quotation mark or quote, then space, then lowercase.
-     *
-     * @param $statement
-     * @return bool
-     */
-    private static function isEndQuote($statement)
-    {
-        $trimmed = Multibyte::trim($statement);
-        $first = mb_substr($statement, 0, 1);
-
-        return in_array($trimmed, ['"', '\''])
-            || (
-                in_array($first, ['"', '\''])
-                && mb_substr($statement, 1, 1) === ' '
-                && ctype_lower(mb_substr($statement, 2, 1)) === true
-            );
-    }
-
-    /**
-     * Merges items into larger sentences.
-     * Multibyte.php safe
-     *
-     * @param string[] $shorts
-     * @return string[]
-     */
-    private function sentenceMerge($shorts)
-    {
-        $non_abbreviating_terminals = array_diff($this->terminals, $this->abbreviators);
-
-        $sentences = [];
-
-        $sentence = '';
-        $has_words = false;
-        $previous_word_ending = null;
-        foreach ($shorts as $short) {
-            $word_count = count(mb_split('\s+', Multibyte::trim($short)));
-            $after_non_abbreviating_terminal = in_array($previous_word_ending, $non_abbreviating_terminals);
-
-            if ($after_non_abbreviating_terminal
-                || ($has_words && $word_count > 1)) {
-
-                $sentences[] = $sentence;
-
-                $sentence = '';
-                $has_words = false;
-            }
-
-            $has_words = $has_words
-                || $word_count > 1;
-
-            $sentence .= $short;
-            $previous_word_ending = mb_substr($short, -1);
-        }
-
-        if (!empty($sentence)) {
-            $sentences[] = $sentence;
-        }
-
-        return $sentences;
-    }
-
-    /**
-     * Return the sentences sentences detected in the provided text.
-     * Set the Sentence::SPLIT_TRIM flag to trim whitespace.
-     * @param string $text
-     * @param integer $flags
-     * @return string[]
-     */
-    public function split($text, $flags = 0)
-    {
-        static $pipeline = [
-            'floatNumberClean',
-            'punctuationSplit',
-            'parenthesesMerge', // also works after punctuationMerge or abbreviationMerge
-            'punctuationMerge',
-            'abbreviationMerge',
-            'closeQuotesMerge',
-            'sentenceMerge',
-            'floatNumberRevert'
-        ];
-
-        // clean funny quotes
-        $text = Multibyte::cleanUnicode($text);
-
-        // Split
-        $sentences = [];
-        foreach (self::linebreakSplit($text) as $input) {
-            if (Multibyte::trim($input) !== '') {
-                foreach ($pipeline as $method) {
-                    $input = $this->$method($input);
-                }
-                $sentences = array_merge($sentences, $input);
-            }
-        }
-
-        // Post process
-        if ($flags & self::SPLIT_TRIM) {
-            return self::trimSentences($sentences);
-        }
-
-        return $sentences;
-    }
-
-    /**
-     * Multibyte.php trim each string in an array.
-     * @param string[] $sentences
-     * @return string[]
-     */
-    private static function trimSentences($sentences)
-    {
-        return array_map(function ($sentence) {
-            return Multibyte::trim($sentence);
-        }, $sentences);
-    }
-
-    /**
-     * Return the number of sentences detected in the provided text.
-     * @param string $text
-     * @return integer
-     */
-    public function count($text)
-    {
-        return count($this->split($text));
-    }
+		return array_map(function($value) {
+			foreach ($this->floatNumbers as $number => $hash) {
+				$value = str_replace($hash, $number, $value);
+			}
+			return $value;
+		}, $text);
+	}
+
+	/**
+	 * Breaks a piece of text into lines by linebreak.
+	 * Eats up any linebreak characters as if one.
+	 *
+	 * Multibyte.php safe
+	 *
+	 * @param string $text
+	 * @return string[]
+	 */
+	private static function linebreakSplit($text)
+	{
+		$lines = [];
+		$line = '';
+
+		foreach (Multibyte::split('([\r\n]+)', $text, -1, PREG_SPLIT_DELIM_CAPTURE) as $part) {
+			$line .= $part;
+			if (Multibyte::trim($part) === '') {
+				$lines[] = $line;
+				$line = '';
+			}
+		}
+		$lines[] = $line;
+
+		return $lines;
+	}
+
+	/**
+	 * Splits an array of lines by (consecutive sequences of)
+	 * terminals, keeping terminals.
+	 *
+	 * Multibyte.php safe (atleast for UTF-8)
+	 *
+	 * For example:
+	 *    "There ... is. More!"
+	 *        ... becomes ...
+	 *    [ "There ", "...", " is", ".", " More", "!" ]
+	 *
+	 * @param string $line
+	 * @return string[]
+	 */
+	private function punctuationSplit($line)
+	{
+		$parts = [];
+
+		$chars = preg_split('//u', $line, -1, PREG_SPLIT_NO_EMPTY); // This is UTF8 multibyte safe!
+		$is_terminal = in_array($chars[0], $this->terminals);
+
+		$part = '';
+		foreach ($chars as $index => $char) {
+			if (in_array($char, $this->terminals) !== $is_terminal) {
+				$parts[] = $part;
+				$part = '';
+				$is_terminal = !$is_terminal;
+			}
+			$part .= $char;
+		}
+
+		if (!empty($part)) {
+			$parts[] = $part;
+		}
+
+		return $parts;
+	}
+
+	/**
+	 * Appends each terminal item after it's preceding
+	 * non-terminals.
+	 *
+	 * Multibyte.php safe (atleast for UTF-8)
+	 *
+	 * For example:
+	 *    [ "There ", "...", " is", ".", " More", "!" ]
+	 *        ... becomes ...
+	 *    [ "There ... is.", "More!" ]
+	 *
+	 * @param string[] $punctuations
+	 * @return string[]
+	 */
+	private function punctuationMerge($punctuations)
+	{
+		$definite_terminals = array_diff($this->terminals, $this->abbreviators);
+
+		$merges = [];
+		$merge = '';
+
+		$filtered = array_filter($punctuations, function ($p) {
+			return $p !== '';
+		});
+
+		foreach ($filtered as $punctuation) {
+			$merge .= $punctuation;
+			if (mb_strlen($punctuation) === 1
+				&& in_array($punctuation, $this->terminals)) {
+				$merges[] = $merge;
+				$merge = '';
+			} else {
+				foreach ($definite_terminals as $terminal) {
+					if (mb_strpos($punctuation, $terminal) !== false) {
+						$merges[] = $merge;
+						$merge = '';
+						break;
+					}
+				}
+			}
+		}
+		if (!empty($merge)) {
+			$merges[] = $merge;
+		}
+
+		return $merges;
+	}
+
+	/**
+	 * Looks for capitalized abbreviations & includes them with the following fragment.
+	 *
+	 * For example:
+	 *    [ "Last week, former director of the F.B.I. James B. Comey was fired. Mr. Comey was not available for comment." ]
+	 *        ... becomes ...
+	 *    [ "Last week, former director of the F.B.I. James B. Comey was fired." ]
+	 *  [ "Mr. Comey was not available for comment." ]
+	 *
+	 * @param string[] $fragments
+	 * @return string[]
+	 */
+	private function abbreviationMerge($fragments)
+	{
+		$return_fragment = [];
+
+		$previous_fragment = '';
+		$previous_is_abbreviation = false;
+		$i = 0;
+		foreach ($fragments as $fragment) {
+			$is_abbreviation = self::isAbreviation($fragment);
+
+			// merge previous fragment with this
+			if ($previous_is_abbreviation) {
+				$fragment = $previous_fragment . $fragment;
+			}
+			$return_fragment[$i] = $fragment;
+
+			$previous_is_abbreviation = $is_abbreviation;
+			$previous_fragment = $fragment;
+
+			// only increment if this isn't an abbreviation
+			if (!$is_abbreviation) {
+				$i++;
+			}
+		}
+		return $return_fragment;
+	}
+
+	/**
+	 * Check if the last word of fragment starts with a Capital, ends in "." & has less than 3 characters.
+	 *
+	 * @param $fragment
+	 * @return bool
+	 */
+	private static function isAbreviation($fragment)
+	{
+		$words = mb_split('\s+', Multibyte::trim($fragment));
+
+		$word_count = count($words);
+
+		$last_word = Multibyte::trim($words[$word_count - 1]);
+		$last_is_capital = preg_match('#^\p{Lu}#u', $last_word);
+		$last_is_abbreviation = mb_substr(Multibyte::trim($fragment), -1) === '.';
+
+		return $last_is_capital > 0
+			&& $last_is_abbreviation > 0
+			&& mb_strlen($last_word) <= 3;
+	}
+
+	/**
+	 * Merges any part starting with a closing parenthesis ')' to the previous
+	 * part.
+	 *
+	 * @param string[] $parts
+	 * @return string[]
+	 */
+	private function parenthesesMerge($parts)
+	{
+		$subsentences = [];
+
+		foreach ($parts as $part) {
+			if ($part[0] === ')') {
+				$subsentences[count($subsentences) - 1] .= $part;
+			} else {
+				$subsentences[] = $part;
+			}
+		}
+
+		return $subsentences;
+	}
+
+	/**
+	 * Looks for closing quotes to include them with the previous statement.
+	 * "That was very interesting," he said.
+	 * "That was very interesting."
+	 *
+	 * @param string[] $statements
+	 * @return string[]
+	 */
+	private function closeQuotesMerge($statements)
+	{
+		$i = 0;
+		$previous_statement = '';
+		$return = [];
+		foreach ($statements as $statement) {
+			if (self::isEndQuote($statement)) {
+				$statement = $previous_statement . $statement;
+			} else {
+				$i++;
+			}
+
+			$return[$i] = $statement;
+			$previous_statement = $statement;
+		}
+
+		return $return;
+	}
+
+	/**
+	 * Check if the entire string is a quotation mark or quote, then space, then lowercase.
+	 *
+	 * @param $statement
+	 * @return bool
+	 */
+	private static function isEndQuote($statement)
+	{
+		$trimmed = Multibyte::trim($statement);
+		$first = mb_substr($statement, 0, 1);
+
+		return in_array($trimmed, ['"', '\''])
+			|| (
+				in_array($first, ['"', '\''])
+				&& mb_substr($statement, 1, 1) === ' '
+				&& ctype_lower(mb_substr($statement, 2, 1)) === true
+			);
+	}
+
+	/**
+	 * Merges items into larger sentences.
+	 * Multibyte.php safe
+	 *
+	 * @param string[] $shorts
+	 * @return string[]
+	 */
+	private function sentenceMerge($shorts)
+	{
+		$non_abbreviating_terminals = array_diff($this->terminals, $this->abbreviators);
+
+		$sentences = [];
+
+		$sentence = '';
+		$has_words = false;
+		$previous_word_ending = null;
+		foreach ($shorts as $short) {
+			$word_count = count(mb_split('\s+', Multibyte::trim($short)));
+			$after_non_abbreviating_terminal = in_array($previous_word_ending, $non_abbreviating_terminals);
+
+			if ($after_non_abbreviating_terminal
+				|| ($has_words && $word_count > 1)) {
+
+				$sentences[] = $sentence;
+
+				$sentence = '';
+				$has_words = false;
+			}
+
+			$has_words = $has_words
+				|| $word_count > 1;
+
+			$sentence .= $short;
+			$previous_word_ending = mb_substr($short, -1);
+		}
+
+		if (!empty($sentence)) {
+			$sentences[] = $sentence;
+		}
+
+		return $sentences;
+	}
+
+	/**
+	 * Return the sentences sentences detected in the provided text.
+	 * Set the Sentence::SPLIT_TRIM flag to trim whitespace.
+	 * @param string $text
+	 * @param integer $flags
+	 * @return string[]
+	 */
+	public function split($text, $flags = 0)
+	{
+		static $pipeline = [
+			'floatNumberClean',
+			'punctuationSplit',
+			'parenthesesMerge', // also works after punctuationMerge or abbreviationMerge
+			'punctuationMerge',
+			'abbreviationMerge',
+			'closeQuotesMerge',
+			'sentenceMerge',
+			'floatNumberRevert'
+		];
+
+		// clean funny quotes
+		$text = Multibyte::cleanUnicode($text);
+
+		// Split
+		$sentences = [];
+		foreach (self::linebreakSplit($text) as $input) {
+			if (Multibyte::trim($input) !== '') {
+				foreach ($pipeline as $method) {
+					$input = $this->$method($input);
+				}
+				$sentences = array_merge($sentences, $input);
+			}
+		}
+
+		// Post process
+		if ($flags & self::SPLIT_TRIM) {
+			return self::trimSentences($sentences);
+		}
+
+		return $sentences;
+	}
+
+	/**
+	 * Multibyte.php trim each string in an array.
+	 * @param string[] $sentences
+	 * @return string[]
+	 */
+	private static function trimSentences($sentences)
+	{
+		return array_map(function ($sentence) {
+			return Multibyte::trim($sentence);
+		}, $sentences);
+	}
+
+	/**
+	 * Return the number of sentences detected in the provided text.
+	 * @param string $text
+	 * @return integer
+	 */
+	public function count($text)
+	{
+		return count($this->split($text));
+	}
 
 }

src/Multibyte.php

Indentation +173 added lines, -173 removed lines

@@ -7,177 +7,177 @@
  */
 class Multibyte
 {
-    //https://stackoverflow.com/questions/20025030/convert-all-types-of-smart-quotes-with-php
-    private static $unicodeCharacterMap = [
-        // Windows codepage 1252
-        "\xC2\x82" => "'", // U+0082⇒U+201A single low-9 quotation mark
-        "\xC2\x84" => '"', // U+0084⇒U+201E double low-9 quotation mark
-        "\xC2\x8B" => "'", // U+008B⇒U+2039 single left-pointing angle quotation mark
-        "\xC2\x91" => "'", // U+0091⇒U+2018 left single quotation mark
-        "\xC2\x92" => "'", // U+0092⇒U+2019 right single quotation mark
-        "\xC2\x93" => '"', // U+0093⇒U+201C left double quotation mark
-        "\xC2\x94" => '"', // U+0094⇒U+201D right double quotation mark
-        "\xC2\x9B" => "'", // U+009B⇒U+203A single right-pointing angle quotation mark
-        // Regular Unicode     // U+0022 quotation mark (")
-        // U+0027 apostrophe     (')
-        "\xC2\xAB" => '"', // U+00AB left-pointing double angle quotation mark
-        "\xC2\xBB" => '"', // U+00BB right-pointing double angle quotation mark
-        "\xE2\x80\x98" => "'", // U+2018 left single quotation mark
-        "\xE2\x80\x99" => "'", // U+2019 right single quotation mark
-        "\xE2\x80\x9A" => "'", // U+201A single low-9 quotation mark
-        "\xE2\x80\x9B" => "'", // U+201B single high-reversed-9 quotation mark
-        "\xE2\x80\x9C" => '"', // U+201C left double quotation mark
-        "\xE2\x80\x9D" => '"', // U+201D right double quotation mark
-        "\xE2\x80\x9E" => '"', // U+201E double low-9 quotation mark
-        "\xE2\x80\x9F" => '"', // U+201F double high-reversed-9 quotation mark
-        "\xE2\x80\xB9" => "'", // U+2039 single left-pointing angle quotation mark
-        "\xE2\x80\xBA" => "'", // U+203A single right-pointing angle quotation mark
-    ];
-
-    /**
-     * Replace
-     *
-     * @staticvar array $chr_map
-     * @param string $string
-     * @return string
-     */
-    public static function cleanUnicode($string)
-    {
-        $character = array_keys(self::$unicodeCharacterMap); // but: for efficiency you should
-        $replace = array_values(self::$unicodeCharacterMap); // pre-calculate these two arrays
-        return str_replace($character, $replace, html_entity_decode($string, ENT_QUOTES, "UTF-8"));
-    }
-
-    /**
-     * Multibyte.php safe version of standard trim() function.
-     *
-     * @param string $string
-     * @return string
-     */
-    public static function trim($string)
-    {
-        return mb_ereg_replace('^\s*([\s\S]*?)\s*$', '\1', $string);
-    }
-
-    /**
-     * A cross between mb_split and preg_split, adding the preg_split flags
-     * to mb_split.
-     *
-     * @param string $pattern
-     * @param string $string
-     * @param int $limit
-     * @param int $flags
-     * @return array
-     */
-    public static function split($pattern, $string, $limit = -1, $flags = 0)
-    {
-        $offset_capture = (bool)($flags & PREG_SPLIT_OFFSET_CAPTURE);
-
-        $lengths = self::getSplitLengths($pattern, $string);
-
-        // Substrings
-        $parts = [];
-        $position = 0;
-        $count = 1;
-        foreach ($lengths as $length) {
-            if (self::isLastPart($length, $flags, $limit, $count)) {
-                $parts[] = self::makePart($string, $position, null, $offset_capture);
-                return $parts;
-            }
-
-            if (self::isPart($length, $flags)) {
-                $parts[] = self::makePart($string, $position, $length[0], $offset_capture);
-            }
-
-            $position += $length[0];
-        }
-
-        return $parts;
-    }
-
-    /**
-     * @param $length
-     * @param $flags
-     * @param $limit
-     * @param $count
-     * @return bool
-     */
-    private static function isLastPart($length, $flags, $limit, &$count)
-    {
-        $split_empty = !($flags & PREG_SPLIT_NO_EMPTY) || $length[0];
-        $is_delimiter = $length[1];
-
-        return $limit > 0
-            && !$is_delimiter
-            && $split_empty
-            && ++$count > $limit;
-    }
-
-    /**
-     * @param $length
-     * @param $flags
-     * @return bool
-     */
-    private static function isPart($length, $flags)
-    {
-        $split_empty = !($flags & PREG_SPLIT_NO_EMPTY) || $length[0];
-        $is_delimiter = $length[1];
-        $is_captured = ($flags & PREG_SPLIT_DELIM_CAPTURE) && $length[2];
-
-        return (!$is_delimiter
-                || $is_captured)
-            && $split_empty;
-    }
-
-    /**
-     * Make part
-     * @param string $string
-     * @param integer $position
-     * @param integer|null $length
-     * @param bool $offset_capture
-     * @return array|string
-     */
-    private static function makePart($string, $position, $length = null, $offset_capture = false)
-    {
-        $cut = mb_strcut($string, $position, $length);
-
-        return $offset_capture
-            ? [$cut, $position]
-            : $cut;
-    }
-
-    /**
-     * Splits the string by pattern and for each element (part or split) returns:
-     *  [ 0 => length, 1 => is_delimiter?, 2 =>
-     *
-     * @param $pattern
-     * @param $string
-     * @return array
-     */
-    private static function getSplitLengths($pattern, $string)
-    {
-        $strlen = strlen($string); // bytes!
-        $lengths = [];
-
-        mb_ereg_search_init($string);
-
-        $position = 0;
-        while ($position < $strlen
-            && ($array = mb_ereg_search_pos($pattern, '')) !== false) {
-            // capture split
-            $lengths[] = [$array[0] - $position, false, null];
-
-            // move position
-            $position = $array[0] + $array[1];
-
-            // capture delimiter
-            $regs = mb_ereg_search_getregs();
-            $lengths[] = [$array[1], true, isset($regs[1]) && $regs[1]];
-        }
-
-        // Add last bit, if not ending with split
-        $lengths[] = [$strlen - $position, false, null];
-
-        return $lengths;
-    }
+	//https://stackoverflow.com/questions/20025030/convert-all-types-of-smart-quotes-with-php
+	private static $unicodeCharacterMap = [
+		// Windows codepage 1252
+		"\xC2\x82" => "'", // U+0082⇒U+201A single low-9 quotation mark
+		"\xC2\x84" => '"', // U+0084⇒U+201E double low-9 quotation mark
+		"\xC2\x8B" => "'", // U+008B⇒U+2039 single left-pointing angle quotation mark
+		"\xC2\x91" => "'", // U+0091⇒U+2018 left single quotation mark
+		"\xC2\x92" => "'", // U+0092⇒U+2019 right single quotation mark
+		"\xC2\x93" => '"', // U+0093⇒U+201C left double quotation mark
+		"\xC2\x94" => '"', // U+0094⇒U+201D right double quotation mark
+		"\xC2\x9B" => "'", // U+009B⇒U+203A single right-pointing angle quotation mark
+		// Regular Unicode     // U+0022 quotation mark (")
+		// U+0027 apostrophe     (')
+		"\xC2\xAB" => '"', // U+00AB left-pointing double angle quotation mark
+		"\xC2\xBB" => '"', // U+00BB right-pointing double angle quotation mark
+		"\xE2\x80\x98" => "'", // U+2018 left single quotation mark
+		"\xE2\x80\x99" => "'", // U+2019 right single quotation mark
+		"\xE2\x80\x9A" => "'", // U+201A single low-9 quotation mark
+		"\xE2\x80\x9B" => "'", // U+201B single high-reversed-9 quotation mark
+		"\xE2\x80\x9C" => '"', // U+201C left double quotation mark
+		"\xE2\x80\x9D" => '"', // U+201D right double quotation mark
+		"\xE2\x80\x9E" => '"', // U+201E double low-9 quotation mark
+		"\xE2\x80\x9F" => '"', // U+201F double high-reversed-9 quotation mark
+		"\xE2\x80\xB9" => "'", // U+2039 single left-pointing angle quotation mark
+		"\xE2\x80\xBA" => "'", // U+203A single right-pointing angle quotation mark
+	];
+
+	/**
+	 * Replace
+	 *
+	 * @staticvar array $chr_map
+	 * @param string $string
+	 * @return string
+	 */
+	public static function cleanUnicode($string)
+	{
+		$character = array_keys(self::$unicodeCharacterMap); // but: for efficiency you should
+		$replace = array_values(self::$unicodeCharacterMap); // pre-calculate these two arrays
+		return str_replace($character, $replace, html_entity_decode($string, ENT_QUOTES, "UTF-8"));
+	}
+
+	/**
+	 * Multibyte.php safe version of standard trim() function.
+	 *
+	 * @param string $string
+	 * @return string
+	 */
+	public static function trim($string)
+	{
+		return mb_ereg_replace('^\s*([\s\S]*?)\s*$', '\1', $string);
+	}
+
+	/**
+	 * A cross between mb_split and preg_split, adding the preg_split flags
+	 * to mb_split.
+	 *
+	 * @param string $pattern
+	 * @param string $string
+	 * @param int $limit
+	 * @param int $flags
+	 * @return array
+	 */
+	public static function split($pattern, $string, $limit = -1, $flags = 0)
+	{
+		$offset_capture = (bool)($flags & PREG_SPLIT_OFFSET_CAPTURE);
+
+		$lengths = self::getSplitLengths($pattern, $string);
+
+		// Substrings
+		$parts = [];
+		$position = 0;
+		$count = 1;
+		foreach ($lengths as $length) {
+			if (self::isLastPart($length, $flags, $limit, $count)) {
+				$parts[] = self::makePart($string, $position, null, $offset_capture);
+				return $parts;
+			}
+
+			if (self::isPart($length, $flags)) {
+				$parts[] = self::makePart($string, $position, $length[0], $offset_capture);
+			}
+
+			$position += $length[0];
+		}
+
+		return $parts;
+	}
+
+	/**
+	 * @param $length
+	 * @param $flags
+	 * @param $limit
+	 * @param $count
+	 * @return bool
+	 */
+	private static function isLastPart($length, $flags, $limit, &$count)
+	{
+		$split_empty = !($flags & PREG_SPLIT_NO_EMPTY) || $length[0];
+		$is_delimiter = $length[1];
+
+		return $limit > 0
+			&& !$is_delimiter
+			&& $split_empty
+			&& ++$count > $limit;
+	}
+
+	/**
+	 * @param $length
+	 * @param $flags
+	 * @return bool
+	 */
+	private static function isPart($length, $flags)
+	{
+		$split_empty = !($flags & PREG_SPLIT_NO_EMPTY) || $length[0];
+		$is_delimiter = $length[1];
+		$is_captured = ($flags & PREG_SPLIT_DELIM_CAPTURE) && $length[2];
+
+		return (!$is_delimiter
+				|| $is_captured)
+			&& $split_empty;
+	}
+
+	/**
+	 * Make part
+	 * @param string $string
+	 * @param integer $position
+	 * @param integer|null $length
+	 * @param bool $offset_capture
+	 * @return array|string
+	 */
+	private static function makePart($string, $position, $length = null, $offset_capture = false)
+	{
+		$cut = mb_strcut($string, $position, $length);
+
+		return $offset_capture
+			? [$cut, $position]
+			: $cut;
+	}
+
+	/**
+	 * Splits the string by pattern and for each element (part or split) returns:
+	 *  [ 0 => length, 1 => is_delimiter?, 2 =>
+	 *
+	 * @param $pattern
+	 * @param $string
+	 * @return array
+	 */
+	private static function getSplitLengths($pattern, $string)
+	{
+		$strlen = strlen($string); // bytes!
+		$lengths = [];
+
+		mb_ereg_search_init($string);
+
+		$position = 0;
+		while ($position < $strlen
+			&& ($array = mb_ereg_search_pos($pattern, '')) !== false) {
+			// capture split
+			$lengths[] = [$array[0] - $position, false, null];
+
+			// move position
+			$position = $array[0] + $array[1];
+
+			// capture delimiter
+			$regs = mb_ereg_search_getregs();
+			$lengths[] = [$array[1], true, isset($regs[1]) && $regs[1]];
+		}
+
+		// Add last bit, if not ending with split
+		$lengths[] = [$strlen - $position, false, null];
+
+		return $lengths;
+	}
 }
\ No newline at end of file

Spacing +1 added lines, -1 removed lines

@@ -71,7 +71,7 @@
      */
     public static function split($pattern, $string, $limit = -1, $flags = 0)
     {
-        $offset_capture = (bool)($flags & PREG_SPLIT_OFFSET_CAPTURE);
+        $offset_capture = (bool) ($flags & PREG_SPLIT_OFFSET_CAPTURE);
 
         $lengths = self::getSplitLengths($pattern, $string);