chrislit /
abydos
| Total Complexity | 59 |
| Total Lines | 1024 |
| Duplicated Lines | 9.38 % |
| Changes | 0 | ||
Duplicate code is one of the most pungent code smells. A rule that is often used is to re-structure code once it is duplicated in three or more places.
Common duplication problems, and corresponding solutions are:
Complex classes like tests.test_stemmer often do a lot of different things. To break such a class down, we need to identify a cohesive component within that class. A common approach to find such a component is to look for fields/methods that share the same prefixes, or suffixes.
Once you have determined the fields that belong together, you can apply the Extract Class refactoring. If the component makes sense as a sub-class, Extract Subclass is also a candidate, and is often faster.
| 1 | # -*- coding: utf-8 -*- |
||
| 2 | |||
| 3 | # Copyright 2014-2018 by Christopher C. Little. |
||
| 4 | # This file is part of Abydos. |
||
| 5 | # |
||
| 6 | # Abydos is free software: you can redistribute it and/or modify |
||
| 7 | # it under the terms of the GNU General Public License as published by |
||
| 8 | # the Free Software Foundation, either version 3 of the License, or |
||
| 9 | # (at your option) any later version. |
||
| 10 | # |
||
| 11 | # Abydos is distributed in the hope that it will be useful, |
||
| 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
||
| 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||
| 14 | # GNU General Public License for more details. |
||
| 15 | # |
||
| 16 | # You should have received a copy of the GNU General Public License |
||
| 17 | # along with Abydos. If not, see <http://www.gnu.org/licenses/>. |
||
| 18 | |||
| 19 | """abydos.tests.test_stemmer. |
||
| 20 | |||
| 21 | This module contains unit tests for abydos.stemmer |
||
| 22 | """ |
||
| 23 | |||
| 24 | from __future__ import unicode_literals |
||
| 25 | |||
| 26 | import codecs |
||
| 27 | import os |
||
| 28 | import unittest |
||
| 29 | |||
| 30 | from abydos.stemmer import _ends_in_cvc, _ends_in_doubled_cons, _m_degree, \ |
||
| 31 | _sb_ends_in_short_syllable, _sb_has_vowel, _sb_r1, _sb_r2, \ |
||
| 32 | _sb_short_word, caumanns, clef_german, clef_german_plus, clef_swedish, \ |
||
| 33 | lovins, paice_husk, porter, porter2, sb_danish, sb_dutch, sb_german, \ |
||
| 34 | sb_norwegian, sb_swedish, uealite |
||
| 35 | |||
| 36 | TESTDIR = os.path.dirname(__file__) |
||
| 37 | |||
| 38 | |||
| 39 | class LovinsTestCases(unittest.TestCase): |
||
| 40 | """Test Lovins functions. |
||
| 41 | |||
| 42 | abydos.stemmer.lovins |
||
| 43 | """ |
||
| 44 | |||
| 45 | def test_lovins(self): |
||
| 46 | """Test abydos.stemmer.lovins.""" |
||
| 47 | # base case |
||
| 48 | self.assertEqual(lovins(''), '') |
||
| 49 | |||
| 50 | # test cases from Lovins' "Development of a Stemming Algorithm": |
||
| 51 | # http://www.mt-archive.info/MT-1968-Lovins.pdf |
||
| 52 | self.assertEqual(lovins('magnesia'), 'magnes') |
||
| 53 | self.assertEqual(lovins('magnesite'), 'magnes') |
||
| 54 | self.assertEqual(lovins('magnesian'), 'magnes') |
||
| 55 | self.assertEqual(lovins('magnesium'), 'magnes') |
||
| 56 | self.assertEqual(lovins('magnet'), 'magnet') |
||
| 57 | self.assertEqual(lovins('magnetic'), 'magnet') |
||
| 58 | self.assertEqual(lovins('magneto'), 'magnet') |
||
| 59 | self.assertEqual(lovins('magnetically'), 'magnet') |
||
| 60 | self.assertEqual(lovins('magnetism'), 'magnet') |
||
| 61 | self.assertEqual(lovins('magnetite'), 'magnet') |
||
| 62 | self.assertEqual(lovins('magnetitic'), 'magnet') |
||
| 63 | self.assertEqual(lovins('magnetizable'), 'magnet') |
||
| 64 | self.assertEqual(lovins('magnetization'), 'magnet') |
||
| 65 | self.assertEqual(lovins('magnetize'), 'magnet') |
||
| 66 | self.assertEqual(lovins('magnetometer'), 'magnetometer') |
||
| 67 | self.assertEqual(lovins('magnetometric'), 'magnetometer') |
||
| 68 | self.assertEqual(lovins('magnetometry'), 'magnetometer') |
||
| 69 | self.assertEqual(lovins('magnetomotive'), 'magnetomot') |
||
| 70 | self.assertEqual(lovins('magnetron'), 'magnetron') |
||
| 71 | self.assertEqual(lovins('metal'), 'metal') |
||
| 72 | self.assertEqual(lovins('metall'), 'metal') |
||
| 73 | self.assertEqual(lovins('metallically'), 'metal') |
||
| 74 | self.assertEqual(lovins('metalliferous'), 'metallifer') |
||
| 75 | self.assertEqual(lovins('metallize'), 'metal') |
||
| 76 | self.assertEqual(lovins('metallurgical'), 'metallurg') |
||
| 77 | self.assertEqual(lovins('metallurgy'), 'metallurg') |
||
| 78 | self.assertEqual(lovins('induction'), 'induc') |
||
| 79 | self.assertEqual(lovins('inductance'), 'induc') |
||
| 80 | self.assertEqual(lovins('induced'), 'induc') |
||
| 81 | self.assertEqual(lovins('angular'), 'angl') |
||
| 82 | self.assertEqual(lovins('angle'), 'angl') |
||
| 83 | |||
| 84 | # missed branch test cases |
||
| 85 | self.assertEqual(lovins('feminism'), 'fem') |
||
| 86 | |||
| 87 | def test_lovins_snowball(self): |
||
| 88 | """Test abydos.stemmer.lovins (Snowball testset). |
||
| 89 | |||
| 90 | These test cases are from |
||
| 91 | https://github.com/snowballstem/snowball-data/tree/master/lovins |
||
| 92 | """ |
||
| 93 | # Snowball Lovins test set |
||
| 94 | with codecs.open(TESTDIR+'/corpora/snowball_lovins.csv', |
||
| 95 | encoding='utf-8') as snowball_testset: |
||
| 96 | next(snowball_testset) |
||
| 97 | for line in snowball_testset: |
||
| 98 | if line[0] != '#': |
||
| 99 | line = line.strip().split(',') |
||
| 100 | word, stem = line[0], line[1] |
||
| 101 | self.assertEqual(lovins(word), stem.lower()) |
||
| 102 | |||
| 103 | |||
| 104 | class PorterTestCases(unittest.TestCase): |
||
| 105 | """Test Porter functions. |
||
| 106 | |||
| 107 | abydos.stemmer._m_degree, abydos.stemmer.porter, |
||
| 108 | abydos.stemmer._sb_has_vowel, abydos.stemmer._ends_in_doubled_cons, |
||
| 109 | & abydos.stemmer._ends_in_cvc |
||
| 110 | """ |
||
| 111 | |||
| 112 | def test_m_degree(self): |
||
| 113 | """Test abydos.stemmer._m_degree.""" |
||
| 114 | _vowels = set('aeiouy') |
||
| 115 | # base case |
||
| 116 | self.assertEqual(_m_degree('', _vowels), 0) |
||
| 117 | |||
| 118 | # m==0 |
||
| 119 | self.assertEqual(_m_degree('tr', _vowels), 0) |
||
| 120 | self.assertEqual(_m_degree('ee', _vowels), 0) |
||
| 121 | self.assertEqual(_m_degree('tree', _vowels), 0) |
||
| 122 | self.assertEqual(_m_degree('y', _vowels), 0) |
||
| 123 | self.assertEqual(_m_degree('by', _vowels), 0) |
||
| 124 | |||
| 125 | # m==1 |
||
| 126 | self.assertEqual(_m_degree('trouble', _vowels), 1) |
||
| 127 | self.assertEqual(_m_degree('oats', _vowels), 1) |
||
| 128 | self.assertEqual(_m_degree('trees', _vowels), 1) |
||
| 129 | self.assertEqual(_m_degree('ivy', _vowels), 1) |
||
| 130 | |||
| 131 | # m==2 |
||
| 132 | self.assertEqual(_m_degree('troubles', _vowels), 2) |
||
| 133 | self.assertEqual(_m_degree('private', _vowels), 2) |
||
| 134 | self.assertEqual(_m_degree('oaten', _vowels), 2) |
||
| 135 | self.assertEqual(_m_degree('orrery', _vowels), 2) |
||
| 136 | |||
| 137 | def test_has_vowel(self): |
||
| 138 | """Test abydos.stemmer._has_vowel.""" |
||
| 139 | _vowels = set('aeiouy') |
||
| 140 | # base case |
||
| 141 | self.assertFalse(_sb_has_vowel('', _vowels)) |
||
| 142 | |||
| 143 | # False cases |
||
| 144 | self.assertFalse(_sb_has_vowel('b', _vowels)) |
||
| 145 | self.assertFalse(_sb_has_vowel('c', _vowels)) |
||
| 146 | self.assertFalse(_sb_has_vowel('bc', _vowels)) |
||
| 147 | self.assertFalse(_sb_has_vowel('bcdfghjklmnpqrstvwxYz', _vowels)) |
||
| 148 | self.assertFalse(_sb_has_vowel('Y', _vowels)) |
||
| 149 | |||
| 150 | # True cases |
||
| 151 | self.assertTrue(_sb_has_vowel('a', _vowels)) |
||
| 152 | self.assertTrue(_sb_has_vowel('e', _vowels)) |
||
| 153 | self.assertTrue(_sb_has_vowel('ae', _vowels)) |
||
| 154 | self.assertTrue(_sb_has_vowel('aeiouy', _vowels)) |
||
| 155 | self.assertTrue(_sb_has_vowel('y', _vowels)) |
||
| 156 | |||
| 157 | self.assertTrue(_sb_has_vowel('ade', _vowels)) |
||
| 158 | self.assertTrue(_sb_has_vowel('cad', _vowels)) |
||
| 159 | self.assertTrue(_sb_has_vowel('add', _vowels)) |
||
| 160 | self.assertTrue(_sb_has_vowel('phi', _vowels)) |
||
| 161 | self.assertTrue(_sb_has_vowel('pfy', _vowels)) |
||
| 162 | |||
| 163 | self.assertFalse(_sb_has_vowel('pfY', _vowels)) |
||
| 164 | |||
| 165 | def test_ends_in_doubled_cons(self): |
||
| 166 | """Test abydos.stemmer._ends_in_doubled_cons.""" |
||
| 167 | _vowels = set('aeiouy') |
||
| 168 | # base case |
||
| 169 | self.assertFalse(_ends_in_doubled_cons('', _vowels)) |
||
| 170 | |||
| 171 | # False cases |
||
| 172 | self.assertFalse(_ends_in_doubled_cons('b', _vowels)) |
||
| 173 | self.assertFalse(_ends_in_doubled_cons('c', _vowels)) |
||
| 174 | self.assertFalse(_ends_in_doubled_cons('bc', _vowels)) |
||
| 175 | self.assertFalse(_ends_in_doubled_cons('bcdfghjklmnpqrstvwxYz', |
||
| 176 | _vowels)) |
||
| 177 | self.assertFalse(_ends_in_doubled_cons('Y', _vowels)) |
||
| 178 | self.assertFalse(_ends_in_doubled_cons('a', _vowels)) |
||
| 179 | self.assertFalse(_ends_in_doubled_cons('e', _vowels)) |
||
| 180 | self.assertFalse(_ends_in_doubled_cons('ae', _vowels)) |
||
| 181 | self.assertFalse(_ends_in_doubled_cons('aeiouy', _vowels)) |
||
| 182 | self.assertFalse(_ends_in_doubled_cons('y', _vowels)) |
||
| 183 | self.assertFalse(_ends_in_doubled_cons('ade', _vowels)) |
||
| 184 | self.assertFalse(_ends_in_doubled_cons('cad', _vowels)) |
||
| 185 | self.assertFalse(_ends_in_doubled_cons('phi', _vowels)) |
||
| 186 | self.assertFalse(_ends_in_doubled_cons('pfy', _vowels)) |
||
| 187 | self.assertFalse(_ends_in_doubled_cons('faddy', _vowels)) |
||
| 188 | self.assertFalse(_ends_in_doubled_cons('aiii', _vowels)) |
||
| 189 | self.assertFalse(_ends_in_doubled_cons('ayyy', _vowels)) |
||
| 190 | |||
| 191 | # True cases |
||
| 192 | self.assertTrue(_ends_in_doubled_cons('add', _vowels)) |
||
| 193 | self.assertTrue(_ends_in_doubled_cons('fadd', _vowels)) |
||
| 194 | self.assertTrue(_ends_in_doubled_cons('fadddd', _vowels)) |
||
| 195 | self.assertTrue(_ends_in_doubled_cons('raYY', _vowels)) |
||
| 196 | self.assertTrue(_ends_in_doubled_cons('doll', _vowels)) |
||
| 197 | self.assertTrue(_ends_in_doubled_cons('parr', _vowels)) |
||
| 198 | self.assertTrue(_ends_in_doubled_cons('parrr', _vowels)) |
||
| 199 | self.assertTrue(_ends_in_doubled_cons('bacc', _vowels)) |
||
| 200 | |||
| 201 | def test_ends_in_cvc(self): |
||
| 202 | """Test abydos.stemmer._ends_in_cvc.""" |
||
| 203 | _vowels = set('aeiouy') |
||
| 204 | # base case |
||
| 205 | self.assertFalse(_ends_in_cvc('', _vowels)) |
||
| 206 | |||
| 207 | # False cases |
||
| 208 | self.assertFalse(_ends_in_cvc('b', _vowels)) |
||
| 209 | self.assertFalse(_ends_in_cvc('c', _vowels)) |
||
| 210 | self.assertFalse(_ends_in_cvc('bc', _vowels)) |
||
| 211 | self.assertFalse(_ends_in_cvc('bcdfghjklmnpqrstvwxYz', _vowels)) |
||
| 212 | self.assertFalse(_ends_in_cvc('YYY', _vowels)) |
||
| 213 | self.assertFalse(_ends_in_cvc('ddd', _vowels)) |
||
| 214 | self.assertFalse(_ends_in_cvc('faaf', _vowels)) |
||
| 215 | self.assertFalse(_ends_in_cvc('rare', _vowels)) |
||
| 216 | self.assertFalse(_ends_in_cvc('rhy', _vowels)) |
||
| 217 | |||
| 218 | # True cases |
||
| 219 | self.assertTrue(_ends_in_cvc('dad', _vowels)) |
||
| 220 | self.assertTrue(_ends_in_cvc('phad', _vowels)) |
||
| 221 | self.assertTrue(_ends_in_cvc('faded', _vowels)) |
||
| 222 | self.assertTrue(_ends_in_cvc('maYor', _vowels)) |
||
| 223 | self.assertTrue(_ends_in_cvc('enlil', _vowels)) |
||
| 224 | self.assertTrue(_ends_in_cvc('parer', _vowels)) |
||
| 225 | self.assertTrue(_ends_in_cvc('padres', _vowels)) |
||
| 226 | self.assertTrue(_ends_in_cvc('bacyc', _vowels)) |
||
| 227 | |||
| 228 | # Special case for W, X, & Y |
||
| 229 | self.assertFalse(_ends_in_cvc('craw', _vowels)) |
||
| 230 | self.assertFalse(_ends_in_cvc('max', _vowels)) |
||
| 231 | self.assertFalse(_ends_in_cvc('cray', _vowels)) |
||
| 232 | |||
| 233 | def test_porter(self): |
||
| 234 | """Test abydos.stemmer.porter.""" |
||
| 235 | # base case |
||
| 236 | self.assertEqual(porter(''), '') |
||
| 237 | |||
| 238 | # simple cases |
||
| 239 | self.assertEqual(porter('c'), 'c') |
||
| 240 | self.assertEqual(porter('da'), 'da') |
||
| 241 | self.assertEqual(porter('ad'), 'ad') |
||
| 242 | self.assertEqual(porter('sing'), 'sing') |
||
| 243 | self.assertEqual(porter('singing'), 'sing') |
||
| 244 | |||
| 245 | # missed branch test cases |
||
| 246 | self.assertEqual(porter('capitalism'), 'capit') |
||
| 247 | self.assertEqual(porter('fatalism'), 'fatal') |
||
| 248 | self.assertEqual(porter('stional'), 'stional') |
||
| 249 | self.assertEqual(porter('palism'), 'palism') |
||
| 250 | self.assertEqual(porter('sization'), 'sizat') |
||
| 251 | self.assertEqual(porter('licated'), 'licat') |
||
| 252 | self.assertEqual(porter('lical'), 'lical') |
||
| 253 | |||
| 254 | def test_porter_early_english(self): |
||
| 255 | """Test abydos.stemmer.porter (early English).""" |
||
| 256 | # base case |
||
| 257 | self.assertEqual(porter('', early_english=True), '') |
||
| 258 | |||
| 259 | # simple cases (no different from regular stemmer) |
||
| 260 | self.assertEqual(porter('c', early_english=True), 'c') |
||
| 261 | self.assertEqual(porter('da', early_english=True), 'da') |
||
| 262 | self.assertEqual(porter('ad', early_english=True), 'ad') |
||
| 263 | self.assertEqual(porter('sing', early_english=True), 'sing') |
||
| 264 | self.assertEqual(porter('singing', early_english=True), 'sing') |
||
| 265 | |||
| 266 | # make |
||
| 267 | self.assertEqual(porter('make', early_english=True), 'make') |
||
| 268 | self.assertEqual(porter('makes', early_english=True), 'make') |
||
| 269 | self.assertEqual(porter('maketh', early_english=True), 'make') |
||
| 270 | self.assertEqual(porter('makest', early_english=True), 'make') |
||
| 271 | |||
| 272 | # say |
||
| 273 | self.assertEqual(porter('say', early_english=True), 'sai') |
||
| 274 | self.assertEqual(porter('says', early_english=True), 'sai') |
||
| 275 | self.assertEqual(porter('sayeth', early_english=True), 'sai') |
||
| 276 | self.assertEqual(porter('sayest', early_english=True), 'sai') |
||
| 277 | |||
| 278 | # missed branch test cases |
||
| 279 | self.assertEqual(porter('best', early_english=True), 'best') |
||
| 280 | self.assertEqual(porter('meth', early_english=True), 'meth') |
||
| 281 | |||
| 282 | def test_porter_snowball(self): |
||
| 283 | """Test abydos.stemmer.porter (Snowball testset). |
||
| 284 | |||
| 285 | These test cases are from |
||
| 286 | http://snowball.tartarus.org/algorithms/porter/diffs.txt |
||
| 287 | """ |
||
| 288 | # Snowball Porter test set |
||
| 289 | with open(TESTDIR+'/corpora/snowball_porter.csv') as snowball_testset: |
||
| 290 | next(snowball_testset) |
||
| 291 | for line in snowball_testset: |
||
| 292 | if line[0] != '#': |
||
| 293 | line = line.strip().split(',') |
||
| 294 | word, stem = line[0], line[1] |
||
| 295 | self.assertEqual(porter(word), stem.lower()) |
||
| 296 | |||
| 297 | |||
| 298 | class Porter2TestCases(unittest.TestCase): |
||
| 299 | """Test Porter2 functions. |
||
| 300 | |||
| 301 | abydos.stemmer._sb_r1, abydos.stemmer._sb_r2, |
||
| 302 | abydos.stemmer._sb_ends_in_short_syllable, abydos.stemmer._sb_short_word, |
||
| 303 | & abydos.stemmer.porter2 |
||
| 304 | """ |
||
| 305 | |||
| 306 | def test_sb_r1(self): |
||
| 307 | """Test abydos.stemmer._sb_r1.""" |
||
| 308 | _vowels = set('aeiouy') |
||
| 309 | # base case |
||
| 310 | self.assertEqual(_sb_r1('', _vowels), 0) |
||
| 311 | |||
| 312 | # examples from http://snowball.tartarus.org/texts/r1r2.html |
||
| 313 | self.assertEqual(_sb_r1('beautiful', _vowels), 5) |
||
| 314 | self.assertEqual(_sb_r1('beauty', _vowels), 5) |
||
| 315 | self.assertEqual(_sb_r1('beau', _vowels), 4) |
||
| 316 | self.assertEqual(_sb_r1('animadversion', _vowels), 2) |
||
| 317 | self.assertEqual(_sb_r1('sprinkled', _vowels), 5) |
||
| 318 | self.assertEqual(_sb_r1('eucharist', _vowels), 3) |
||
| 319 | |||
| 320 | def test_sb_r2(self): |
||
| 321 | """Test abydos.stemmer._sb_r2.""" |
||
| 322 | _vowels = set('aeiouy') |
||
| 323 | # base case |
||
| 324 | self.assertEqual(_sb_r2('', _vowels), 0) |
||
| 325 | |||
| 326 | # examples from http://snowball.tartarus.org/texts/r1r2.html |
||
| 327 | self.assertEqual(_sb_r2('beautiful', _vowels), 7) |
||
| 328 | self.assertEqual(_sb_r2('beauty', _vowels), 6) |
||
| 329 | self.assertEqual(_sb_r2('beau', _vowels), 4) |
||
| 330 | self.assertEqual(_sb_r2('animadversion', _vowels), 4) |
||
| 331 | self.assertEqual(_sb_r2('sprinkled', _vowels), 9) |
||
| 332 | self.assertEqual(_sb_r2('eucharist', _vowels), 6) |
||
| 333 | |||
| 334 | def test_sb_ends_in_short_syllable(self): |
||
| 335 | """Test abydos.stemmer._sb_ends_in_short_syllable.""" |
||
| 336 | _vowels = set('aeiouy') |
||
| 337 | _codanonvowels = set('bcdfghjklmnpqrstvz\'') |
||
| 338 | # base case |
||
| 339 | self.assertFalse(_sb_ends_in_short_syllable('', _vowels, |
||
| 340 | _codanonvowels)) |
||
| 341 | |||
| 342 | # examples from |
||
| 343 | # http://snowball.tartarus.org/algorithms/english/stemmer.html |
||
| 344 | self.assertTrue(_sb_ends_in_short_syllable('rap', _vowels, |
||
| 345 | _codanonvowels)) |
||
| 346 | self.assertTrue(_sb_ends_in_short_syllable('trap', _vowels, |
||
| 347 | _codanonvowels)) |
||
| 348 | self.assertTrue(_sb_ends_in_short_syllable('entrap', _vowels, |
||
| 349 | _codanonvowels)) |
||
| 350 | self.assertTrue(_sb_ends_in_short_syllable('ow', _vowels, |
||
| 351 | _codanonvowels)) |
||
| 352 | self.assertTrue(_sb_ends_in_short_syllable('on', _vowels, |
||
| 353 | _codanonvowels)) |
||
| 354 | self.assertTrue(_sb_ends_in_short_syllable('at', _vowels, |
||
| 355 | _codanonvowels)) |
||
| 356 | self.assertFalse(_sb_ends_in_short_syllable('uproot', _vowels, |
||
| 357 | _codanonvowels)) |
||
| 358 | self.assertFalse(_sb_ends_in_short_syllable('uproot', _vowels, |
||
| 359 | _codanonvowels)) |
||
| 360 | self.assertFalse(_sb_ends_in_short_syllable('bestow', _vowels, |
||
| 361 | _codanonvowels)) |
||
| 362 | self.assertFalse(_sb_ends_in_short_syllable('disturb', _vowels, |
||
| 363 | _codanonvowels)) |
||
| 364 | |||
| 365 | # missed branch test cases |
||
| 366 | self.assertFalse(_sb_ends_in_short_syllable('d', _vowels, |
||
| 367 | _codanonvowels)) |
||
| 368 | self.assertFalse(_sb_ends_in_short_syllable('a', _vowels, |
||
| 369 | _codanonvowels)) |
||
| 370 | |||
| 371 | def test_sb_short_word(self): |
||
| 372 | """Test abydos.stemmer._sb_short_word.""" |
||
| 373 | _vowels = set('aeiouy') |
||
| 374 | _codanonvowels = set('bcdfghjklmnpqrstvz\'') |
||
| 375 | # base case |
||
| 376 | self.assertFalse(_sb_short_word('', _vowels, _codanonvowels)) |
||
| 377 | |||
| 378 | # examples from |
||
| 379 | # http://snowball.tartarus.org/algorithms/english/stemmer.html |
||
| 380 | self.assertTrue(_sb_short_word('bed', _vowels, _codanonvowels)) |
||
| 381 | self.assertTrue(_sb_short_word('shed', _vowels, _codanonvowels)) |
||
| 382 | self.assertTrue(_sb_short_word('shred', _vowels, _codanonvowels)) |
||
| 383 | self.assertFalse(_sb_short_word('bead', _vowels, _codanonvowels)) |
||
| 384 | self.assertFalse(_sb_short_word('embed', _vowels, _codanonvowels)) |
||
| 385 | self.assertFalse(_sb_short_word('beds', _vowels, _codanonvowels)) |
||
| 386 | |||
| 387 | def test_porter2(self): |
||
| 388 | """Test abydos.stemmer.porter2.""" |
||
| 389 | # base case |
||
| 390 | self.assertEqual(porter2(''), '') |
||
| 391 | |||
| 392 | # simple cases |
||
| 393 | self.assertEqual(porter2('c'), 'c') |
||
| 394 | self.assertEqual(porter2('da'), 'da') |
||
| 395 | self.assertEqual(porter2('ad'), 'ad') |
||
| 396 | self.assertEqual(porter2('sing'), 'sing') |
||
| 397 | self.assertEqual(porter2('singing'), 'sing') |
||
| 398 | |||
| 399 | # missed branch test cases |
||
| 400 | self.assertEqual(porter2('capitalism'), 'capit') |
||
| 401 | self.assertEqual(porter2('fatalism'), 'fatal') |
||
| 402 | self.assertEqual(porter2('dog\'s'), 'dog') |
||
| 403 | self.assertEqual(porter2('A\'s\''), 'a') |
||
| 404 | self.assertEqual(porter2('agreedly'), 'agre') |
||
| 405 | self.assertEqual(porter2('feedly'), 'feed') |
||
| 406 | self.assertEqual(porter2('stional'), 'stional') |
||
| 407 | self.assertEqual(porter2('palism'), 'palism') |
||
| 408 | self.assertEqual(porter2('sization'), 'sizat') |
||
| 409 | self.assertEqual(porter2('licated'), 'licat') |
||
| 410 | self.assertEqual(porter2('lical'), 'lical') |
||
| 411 | self.assertEqual(porter2('clessly'), 'clessli') |
||
| 412 | self.assertEqual(porter2('tably'), 'tabli') |
||
| 413 | self.assertEqual(porter2('sizer'), 'sizer') |
||
| 414 | self.assertEqual(porter2('livity'), 'liviti') |
||
| 415 | |||
| 416 | def test_porter2_early_english(self): |
||
| 417 | """Test abydos.stemmer.porter2 (early English).""" |
||
| 418 | # base case |
||
| 419 | self.assertEqual(porter2('', early_english=True), '') |
||
| 420 | |||
| 421 | # simple cases (no different from regular stemmer) |
||
| 422 | self.assertEqual(porter2('c', early_english=True), 'c') |
||
| 423 | self.assertEqual(porter2('da', early_english=True), 'da') |
||
| 424 | self.assertEqual(porter2('ad', early_english=True), 'ad') |
||
| 425 | self.assertEqual(porter2('sing', early_english=True), 'sing') |
||
| 426 | self.assertEqual(porter2('singing', early_english=True), 'sing') |
||
| 427 | |||
| 428 | # make |
||
| 429 | self.assertEqual(porter2('make', early_english=True), 'make') |
||
| 430 | self.assertEqual(porter2('makes', early_english=True), 'make') |
||
| 431 | self.assertEqual(porter2('maketh', early_english=True), 'make') |
||
| 432 | self.assertEqual(porter2('makest', early_english=True), 'make') |
||
| 433 | |||
| 434 | # say |
||
| 435 | self.assertEqual(porter2('say', early_english=True), 'say') |
||
| 436 | self.assertEqual(porter2('says', early_english=True), 'say') |
||
| 437 | self.assertEqual(porter2('sayeth', early_english=True), 'say') |
||
| 438 | self.assertEqual(porter2('sayest', early_english=True), 'say') |
||
| 439 | |||
| 440 | # missed branch test cases |
||
| 441 | self.assertEqual(porter2('best', early_english=True), 'best') |
||
| 442 | self.assertEqual(porter2('meth', early_english=True), 'meth') |
||
| 443 | |||
| 444 | def test_porter2_snowball(self): |
||
| 445 | """Test abydos.stemmer.porter2 (Snowball testset). |
||
| 446 | |||
| 447 | These test cases are from |
||
| 448 | http://snowball.tartarus.org/algorithms/english/diffs.txt |
||
| 449 | """ |
||
| 450 | # Snowball Porter test set |
||
| 451 | with open(TESTDIR+'/corpora/snowball_porter2.csv') as snowball_testset: |
||
| 452 | next(snowball_testset) |
||
| 453 | for line in snowball_testset: |
||
| 454 | if line[0] != '#': |
||
| 455 | line = line.strip().split(',') |
||
| 456 | word, stem = line[0], line[1] |
||
| 457 | self.assertEqual(porter2(word), stem.lower()) |
||
| 458 | |||
| 459 | |||
| 460 | class SnowballTestCases(unittest.TestCase): |
||
| 461 | """Test Snowball functions. |
||
| 462 | |||
| 463 | abydos.stemmer.sb_german, abydos.stemmer.sb_dutch, |
||
| 464 | abydos.stemmer.sb_norwegian, abydos.stemmer.sb_swedish, & |
||
| 465 | abydos.stemmer.sb_danish |
||
| 466 | """ |
||
| 467 | |||
| 468 | View Code Duplication | def test_sb_german_snowball(self): |
|
|
Duplication
introduced
by
|
|||
| 469 | """Test abydos.stemmer.sb_german (Snowball testset). |
||
| 470 | |||
| 471 | These test cases are from |
||
| 472 | http://snowball.tartarus.org/algorithms/german/diffs.txt |
||
| 473 | """ |
||
| 474 | # base case |
||
| 475 | self.assertEqual(sb_german(''), '') |
||
| 476 | |||
| 477 | # Snowball German test set |
||
| 478 | with codecs.open(TESTDIR+'/corpora/snowball_german.csv', |
||
| 479 | encoding='utf-8') as snowball_testset: |
||
| 480 | next(snowball_testset) |
||
| 481 | for line in snowball_testset: |
||
| 482 | if line[0] != '#': |
||
| 483 | line = line.strip().split(',') |
||
| 484 | word, stem = line[0], line[1] |
||
| 485 | self.assertEqual(sb_german(word), stem.lower()) |
||
| 486 | |||
| 487 | # missed branch test cases |
||
| 488 | self.assertEqual(sb_german('ikeit'), 'ikeit') |
||
| 489 | |||
| 490 | def test_sb_german_snowball_alt(self): |
||
| 491 | """Test abydos.stemmer.sb_german (alternate vowels).""" |
||
| 492 | # base case |
||
| 493 | self.assertEqual(sb_german('', alternate_vowels=True), '') |
||
| 494 | |||
| 495 | # dämmerung,dammer |
||
| 496 | self.assertEqual(sb_german('dämmerung', alternate_vowels=True), |
||
| 497 | 'dammer') |
||
| 498 | self.assertEqual(sb_german('daemmerung', alternate_vowels=True), |
||
| 499 | 'dammer') |
||
| 500 | self.assertEqual(sb_german('dämmerung'), 'dammer') |
||
| 501 | self.assertEqual(sb_german('daemmerung'), 'daemmer') |
||
| 502 | |||
| 503 | # brötchen,brotch |
||
| 504 | self.assertEqual(sb_german('brötchen', alternate_vowels=True), |
||
| 505 | 'brotch') |
||
| 506 | self.assertEqual(sb_german('broetchen', alternate_vowels=True), |
||
| 507 | 'brotch') |
||
| 508 | self.assertEqual(sb_german('brötchen'), 'brotch') |
||
| 509 | self.assertEqual(sb_german('broetchen'), 'broetch') |
||
| 510 | |||
| 511 | # büro,buro |
||
| 512 | self.assertEqual(sb_german('büro', alternate_vowels=True), 'buro') |
||
| 513 | self.assertEqual(sb_german('buero', alternate_vowels=True), 'buro') |
||
| 514 | self.assertEqual(sb_german('büro'), 'buro') |
||
| 515 | self.assertEqual(sb_german('buero'), 'buero') |
||
| 516 | |||
| 517 | # häufen,hauf |
||
| 518 | self.assertEqual(sb_german('häufen', alternate_vowels=True), 'hauf') |
||
| 519 | self.assertEqual(sb_german('haeufen', alternate_vowels=True), 'hauf') |
||
| 520 | self.assertEqual(sb_german('häufen'), 'hauf') |
||
| 521 | self.assertEqual(sb_german('haeufen'), 'haeuf') |
||
| 522 | |||
| 523 | # quelle,quell |
||
| 524 | self.assertEqual(sb_german('qülle', alternate_vowels=True), 'qull') |
||
| 525 | self.assertEqual(sb_german('quelle', alternate_vowels=True), 'quell') |
||
| 526 | self.assertEqual(sb_german('qülle'), 'qull') |
||
| 527 | self.assertEqual(sb_german('quelle'), 'quell') |
||
| 528 | |||
| 529 | # feuer,feuer |
||
| 530 | self.assertEqual(sb_german('feür', alternate_vowels=True), 'feur') |
||
| 531 | self.assertEqual(sb_german('feuer', alternate_vowels=True), 'feu') |
||
| 532 | self.assertEqual(sb_german('feür'), 'feur') |
||
| 533 | self.assertEqual(sb_german('feuer'), 'feu') |
||
| 534 | |||
| 535 | # über,uber |
||
| 536 | self.assertEqual(sb_german('über', alternate_vowels=True), 'uber') |
||
| 537 | self.assertEqual(sb_german('ueber', alternate_vowels=True), 'uber') |
||
| 538 | self.assertEqual(sb_german('über'), 'uber') |
||
| 539 | self.assertEqual(sb_german('ueber'), 'ueb') |
||
| 540 | |||
| 541 | View Code Duplication | def test_sb_dutch_snowball(self): |
|
| 542 | """Test abydos.stemmer.sb_dutch (Snowball testset). |
||
| 543 | |||
| 544 | These test cases are from |
||
| 545 | http://snowball.tartarus.org/algorithms/dutch/diffs.txt |
||
| 546 | """ |
||
| 547 | # base case |
||
| 548 | self.assertEqual(sb_dutch(''), '') |
||
| 549 | |||
| 550 | # Snowball Dutch test set |
||
| 551 | with codecs.open(TESTDIR+'/corpora/snowball_dutch.csv', |
||
| 552 | encoding='utf-8') as snowball_testset: |
||
| 553 | next(snowball_testset) |
||
| 554 | for line in snowball_testset: |
||
| 555 | if line[0] != '#': |
||
| 556 | line = line.strip().split(',') |
||
| 557 | word, stem = line[0], line[1] |
||
| 558 | self.assertEqual(sb_dutch(word), stem.lower()) |
||
| 559 | |||
| 560 | # missed branch test cases |
||
| 561 | self.assertEqual(sb_dutch('zondulielijk'), 'zondulie') |
||
| 562 | |||
| 563 | View Code Duplication | def test_sb_norwegian_snowball(self): |
|
| 564 | """Test abydos.stemmer.sb_norwegian (Snowball testset). |
||
| 565 | |||
| 566 | These test cases are from |
||
| 567 | http://snowball.tartarus.org/algorithms/norwegian/diffs.txt |
||
| 568 | """ |
||
| 569 | # base case |
||
| 570 | self.assertEqual(sb_norwegian(''), '') |
||
| 571 | |||
| 572 | # Snowball Norwegian test set |
||
| 573 | with codecs.open(TESTDIR+'/corpora/snowball_norwegian.csv', |
||
| 574 | encoding='utf-8') as snowball_testset: |
||
| 575 | next(snowball_testset) |
||
| 576 | for line in snowball_testset: |
||
| 577 | if line[0] != '#': |
||
| 578 | line = line.strip().split(',') |
||
| 579 | word, stem = line[0], line[1] |
||
| 580 | self.assertEqual(sb_norwegian(word), stem.lower()) |
||
| 581 | |||
| 582 | View Code Duplication | def test_sb_swedish_snowball(self): |
|
| 583 | """Test abydos.stemmer.sb_swedish (Snowball testset). |
||
| 584 | |||
| 585 | These test cases are from |
||
| 586 | http://snowball.tartarus.org/algorithms/swedish/diffs.txt |
||
| 587 | """ |
||
| 588 | # base case |
||
| 589 | self.assertEqual(sb_swedish(''), '') |
||
| 590 | |||
| 591 | # Snowball Swedish test set |
||
| 592 | with codecs.open(TESTDIR+'/corpora/snowball_swedish.csv', |
||
| 593 | encoding='utf-8') as snowball_testset: |
||
| 594 | next(snowball_testset) |
||
| 595 | for line in snowball_testset: |
||
| 596 | if line[0] != '#': |
||
| 597 | line = line.strip().split(',') |
||
| 598 | word, stem = line[0], line[1] |
||
| 599 | self.assertEqual(sb_swedish(word), stem.lower()) |
||
| 600 | |||
| 601 | View Code Duplication | def test_sb_danish_snowball(self): |
|
| 602 | """Test abydos.stemmer.sb_danish (Snowball testset). |
||
| 603 | |||
| 604 | These test cases are from |
||
| 605 | http://snowball.tartarus.org/algorithms/danish/diffs.txt |
||
| 606 | """ |
||
| 607 | # base case |
||
| 608 | self.assertEqual(sb_danish(''), '') |
||
| 609 | |||
| 610 | # Snowball Danish test set |
||
| 611 | with codecs.open(TESTDIR+'/corpora/snowball_danish.csv', |
||
| 612 | encoding='utf-8') as snowball_testset: |
||
| 613 | next(snowball_testset) |
||
| 614 | for line in snowball_testset: |
||
| 615 | if line[0] != '#': |
||
| 616 | line = line.strip().split(',') |
||
| 617 | word, stem = line[0], line[1] |
||
| 618 | self.assertEqual(sb_danish(word), stem.lower()) |
||
| 619 | |||
| 620 | |||
| 621 | class CLEFTestCases(unittest.TestCase): |
||
| 622 | """Test CLEF functions. |
||
| 623 | |||
| 624 | abydos.stemmer.clef_german, abydos.stemmer.clef_german_plus, & |
||
| 625 | abydos.stemmer.clef_swedish |
||
| 626 | """ |
||
| 627 | |||
| 628 | def test_clef_german(self): |
||
| 629 | """Test abydos.stemmer.clef_german.""" |
||
| 630 | # base case |
||
| 631 | self.assertEqual(clef_german(''), '') |
||
| 632 | |||
| 633 | # len <= 2 |
||
| 634 | self.assertEqual(clef_german('ä'), 'a') |
||
| 635 | self.assertEqual(clef_german('er'), 'er') |
||
| 636 | self.assertEqual(clef_german('es'), 'es') |
||
| 637 | self.assertEqual(clef_german('äh'), 'ah') |
||
| 638 | |||
| 639 | # len > 2 |
||
| 640 | self.assertEqual(clef_german('deinen'), 'dein') |
||
| 641 | self.assertEqual(clef_german('können'), 'konn') |
||
| 642 | self.assertEqual(clef_german('Damen'), 'dame') |
||
| 643 | self.assertEqual(clef_german('kleines'), 'klein') |
||
| 644 | self.assertEqual(clef_german('Namen'), 'name') |
||
| 645 | self.assertEqual(clef_german('Äpfel'), 'apfel') |
||
| 646 | self.assertEqual(clef_german('Jahre'), 'jahr') |
||
| 647 | self.assertEqual(clef_german('Mannes'), 'mann') |
||
| 648 | self.assertEqual(clef_german('Häuser'), 'haus') |
||
| 649 | self.assertEqual(clef_german('Motoren'), 'motor') |
||
| 650 | self.assertEqual(clef_german('kleine'), 'klein') |
||
| 651 | self.assertEqual(clef_german('Pfingsten'), 'pfingst') |
||
| 652 | self.assertEqual(clef_german('lautest'), 'lautest') |
||
| 653 | self.assertEqual(clef_german('lauteste'), 'lautest') |
||
| 654 | self.assertEqual(clef_german('lautere'), 'lauter') |
||
| 655 | self.assertEqual(clef_german('lautste'), 'lautst') |
||
| 656 | self.assertEqual(clef_german('kleinen'), 'klei') |
||
| 657 | |||
| 658 | def test_clef_german_plus(self): |
||
| 659 | """Test abydos.stemmer.clef_german_plus.""" |
||
| 660 | # base case |
||
| 661 | self.assertEqual(clef_german_plus(''), '') |
||
| 662 | |||
| 663 | # len <= 2 |
||
| 664 | self.assertEqual(clef_german_plus('ä'), 'a') |
||
| 665 | self.assertEqual(clef_german_plus('er'), 'er') |
||
| 666 | self.assertEqual(clef_german_plus('es'), 'es') |
||
| 667 | self.assertEqual(clef_german_plus('äh'), 'ah') |
||
| 668 | |||
| 669 | # len > 2 |
||
| 670 | self.assertEqual(clef_german_plus('deinen'), 'dein') |
||
| 671 | self.assertEqual(clef_german_plus('können'), 'konn') |
||
| 672 | self.assertEqual(clef_german_plus('Damen'), 'dam') |
||
| 673 | self.assertEqual(clef_german_plus('kleines'), 'klein') |
||
| 674 | self.assertEqual(clef_german_plus('Namen'), 'nam') |
||
| 675 | self.assertEqual(clef_german_plus('Äpfel'), 'apfel') |
||
| 676 | self.assertEqual(clef_german_plus('Jahre'), 'jahr') |
||
| 677 | self.assertEqual(clef_german_plus('Mannes'), 'mann') |
||
| 678 | self.assertEqual(clef_german_plus('Häuser'), 'haus') |
||
| 679 | self.assertEqual(clef_german_plus('Motoren'), 'motor') |
||
| 680 | self.assertEqual(clef_german_plus('kleine'), 'klein') |
||
| 681 | self.assertEqual(clef_german_plus('Pfingsten'), 'pfing') |
||
| 682 | self.assertEqual(clef_german_plus('lautest'), 'laut') |
||
| 683 | self.assertEqual(clef_german_plus('lauteste'), 'laut') |
||
| 684 | self.assertEqual(clef_german_plus('lautere'), 'laut') |
||
| 685 | self.assertEqual(clef_german_plus('lautste'), 'laut') |
||
| 686 | self.assertEqual(clef_german_plus('kleinen'), 'klein') |
||
| 687 | self.assertEqual(clef_german_plus('Pfarrern'), 'pfarr') |
||
| 688 | |||
| 689 | def test_clef_swedish(self): |
||
| 690 | """Test abydos.stemmer.clef_swedish.""" |
||
| 691 | # base case |
||
| 692 | self.assertEqual(clef_swedish(''), '') |
||
| 693 | |||
| 694 | # unstemmed |
||
| 695 | self.assertEqual(clef_swedish('konung'), 'konung') |
||
| 696 | |||
| 697 | # len <= 3 |
||
| 698 | self.assertEqual(clef_swedish('km'), 'km') |
||
| 699 | self.assertEqual(clef_swedish('ja'), 'ja') |
||
| 700 | self.assertEqual(clef_swedish('de'), 'de') |
||
| 701 | self.assertEqual(clef_swedish('in'), 'in') |
||
| 702 | self.assertEqual(clef_swedish('a'), 'a') |
||
| 703 | self.assertEqual(clef_swedish('mer'), 'mer') |
||
| 704 | self.assertEqual(clef_swedish('s'), 's') |
||
| 705 | self.assertEqual(clef_swedish('e'), 'e') |
||
| 706 | self.assertEqual(clef_swedish('oss'), 'oss') |
||
| 707 | self.assertEqual(clef_swedish('hos'), 'hos') |
||
| 708 | |||
| 709 | # genitive |
||
| 710 | self.assertEqual(clef_swedish('svenskars'), 'svensk') |
||
| 711 | self.assertEqual(clef_swedish('stadens'), 'stad') |
||
| 712 | self.assertEqual(clef_swedish('kommuns'), 'kommu') |
||
| 713 | self.assertEqual(clef_swedish('aftonbladets'), 'aftonblad') |
||
| 714 | |||
| 715 | # len > 7 |
||
| 716 | self.assertEqual(clef_swedish('fängelser'), 'fäng') |
||
| 717 | self.assertEqual(clef_swedish('möjligheten'), 'möjlig') |
||
| 718 | |||
| 719 | # len > 6 |
||
| 720 | self.assertEqual(clef_swedish('svenskar'), 'svensk') |
||
| 721 | self.assertEqual(clef_swedish('myndigheterna'), 'myndighet') |
||
| 722 | self.assertEqual(clef_swedish('avgörande'), 'avgör') |
||
| 723 | self.assertEqual(clef_swedish('fängelse'), 'fäng') |
||
| 724 | self.assertEqual(clef_swedish('viktigaste'), 'viktig') |
||
| 725 | self.assertEqual(clef_swedish('kvinnorna'), 'kvinn') |
||
| 726 | self.assertEqual(clef_swedish('åklagaren'), 'åklag') |
||
| 727 | |||
| 728 | # len > 5 |
||
| 729 | self.assertEqual(clef_swedish('tidigare'), 'tidig') |
||
| 730 | self.assertEqual(clef_swedish('senast'), 'sen') |
||
| 731 | self.assertEqual(clef_swedish('möjlighet'), 'möjlig') |
||
| 732 | |||
| 733 | # len > 4 |
||
| 734 | self.assertEqual(clef_swedish('svenskar'), 'svensk') |
||
| 735 | self.assertEqual(clef_swedish('skriver'), 'skriv') |
||
| 736 | self.assertEqual(clef_swedish('människor'), 'människ') |
||
| 737 | self.assertEqual(clef_swedish('staden'), 'stad') |
||
| 738 | self.assertEqual(clef_swedish('kunnat'), 'kunn') |
||
| 739 | self.assertEqual(clef_swedish('samarbete'), 'samarbe') |
||
| 740 | self.assertEqual(clef_swedish('aftonbladet'), 'aftonblad') |
||
| 741 | |||
| 742 | # len > 3 |
||
| 743 | self.assertEqual(clef_swedish('allt'), 'all') |
||
| 744 | self.assertEqual(clef_swedish('vilka'), 'vilk') |
||
| 745 | self.assertEqual(clef_swedish('länge'), 'läng') |
||
| 746 | self.assertEqual(clef_swedish('kommun'), 'kommu') |
||
| 747 | |||
| 748 | |||
| 749 | class CaumannsTestCases(unittest.TestCase): |
||
| 750 | """Test Caumanns functions. |
||
| 751 | |||
| 752 | abydos.stemmer.caumanns |
||
| 753 | """ |
||
| 754 | |||
| 755 | def test_caumanns(self): |
||
| 756 | """Test abydos.stemmer.caumanns.""" |
||
| 757 | # base case |
||
| 758 | self.assertEqual(caumanns(''), '') |
||
| 759 | |||
| 760 | # tests from Caumanns' description of the algorithm |
||
| 761 | self.assertEqual(caumanns('singt'), 'sing') |
||
| 762 | self.assertEqual(caumanns('singen'), 'sing') |
||
| 763 | self.assertEqual(caumanns('beliebt'), 'belieb') |
||
| 764 | self.assertEqual(caumanns('beliebtester'), 'belieb') |
||
| 765 | self.assertEqual(caumanns('stören'), 'stor') |
||
| 766 | self.assertEqual(caumanns('stöhnen'), 'stoh') |
||
| 767 | self.assertEqual(caumanns('Kuß'), 'kuss') |
||
| 768 | self.assertEqual(caumanns('Küsse'), 'kuss') |
||
| 769 | self.assertEqual(caumanns('Verlierer'), 'verlier') |
||
| 770 | self.assertEqual(caumanns('Verlies'), 'verlie') |
||
| 771 | self.assertEqual(caumanns('Maus'), 'mau') |
||
| 772 | self.assertEqual(caumanns('Mauer'), 'mau') |
||
| 773 | self.assertEqual(caumanns('Störsender'), 'stor') |
||
| 774 | |||
| 775 | # additional tests to achieve full coverage |
||
| 776 | self.assertEqual(caumanns('Müllerinnen'), 'mullerin') |
||
| 777 | self.assertEqual(caumanns('Matrix'), 'matrix') |
||
| 778 | self.assertEqual(caumanns('Matrizen'), 'matrix') |
||
| 779 | |||
| 780 | def test_caumanns_lucene(self): |
||
| 781 | """Test abydos.stemmer.caumanns (Lucene tests). |
||
| 782 | |||
| 783 | Based on tests from |
||
| 784 | https://svn.apache.org/repos/asf/lucene.net/trunk/test/contrib/Analyzers/De/data.txt |
||
| 785 | This is presumably Apache-licensed. |
||
| 786 | """ |
||
| 787 | # German special characters are replaced: |
||
| 788 | self.assertEqual(caumanns('häufig'), 'haufig') |
||
| 789 | self.assertEqual(caumanns('üor'), 'uor') |
||
| 790 | self.assertEqual(caumanns('björk'), 'bjork') |
||
| 791 | |||
| 792 | # here the stemmer works okay, it maps related words to the same stem: |
||
| 793 | self.assertEqual(caumanns('abschließen'), 'abschliess') |
||
| 794 | self.assertEqual(caumanns('abschließender'), 'abschliess') |
||
| 795 | self.assertEqual(caumanns('abschließendes'), 'abschliess') |
||
| 796 | self.assertEqual(caumanns('abschließenden'), 'abschliess') |
||
| 797 | |||
| 798 | self.assertEqual(caumanns('Tisch'), 'tisch') |
||
| 799 | self.assertEqual(caumanns('Tische'), 'tisch') |
||
| 800 | self.assertEqual(caumanns('Tischen'), 'tisch') |
||
| 801 | self.assertEqual(caumanns('geheimtür'), 'geheimtur') |
||
| 802 | |||
| 803 | self.assertEqual(caumanns('Haus'), 'hau') |
||
| 804 | self.assertEqual(caumanns('Hauses'), 'hau') |
||
| 805 | self.assertEqual(caumanns('Häuser'), 'hau') |
||
| 806 | self.assertEqual(caumanns('Häusern'), 'hau') |
||
| 807 | # here's a case where overstemming occurs, i.e. a word is |
||
| 808 | # mapped to the same stem as unrelated words: |
||
| 809 | self.assertEqual(caumanns('hauen'), 'hau') |
||
| 810 | |||
| 811 | # here's a case where understemming occurs, i.e. two related words |
||
| 812 | # are not mapped to the same stem. This is the case with basically |
||
| 813 | # all irregular forms: |
||
| 814 | self.assertEqual(caumanns('Drama'), 'drama') |
||
| 815 | self.assertEqual(caumanns('Dramen'), 'dram') |
||
| 816 | |||
| 817 | # replace "ß" with 'ss': |
||
| 818 | self.assertEqual(caumanns('Ausmaß'), 'ausmass') |
||
| 819 | |||
| 820 | # fake words to test if suffixes are cut off: |
||
| 821 | self.assertEqual(caumanns('xxxxxe'), 'xxxxx') |
||
| 822 | self.assertEqual(caumanns('xxxxxs'), 'xxxxx') |
||
| 823 | self.assertEqual(caumanns('xxxxxn'), 'xxxxx') |
||
| 824 | self.assertEqual(caumanns('xxxxxt'), 'xxxxx') |
||
| 825 | self.assertEqual(caumanns('xxxxxem'), 'xxxxx') |
||
| 826 | self.assertEqual(caumanns('xxxxxer'), 'xxxxx') |
||
| 827 | self.assertEqual(caumanns('xxxxxnd'), 'xxxxx') |
||
| 828 | # the suffixes are also removed when combined: |
||
| 829 | self.assertEqual(caumanns('xxxxxetende'), 'xxxxx') |
||
| 830 | |||
| 831 | # words that are shorter than four charcters are not changed: |
||
| 832 | self.assertEqual(caumanns('xxe'), 'xxe') |
||
| 833 | # -em and -er are not removed from words shorter than five characters: |
||
| 834 | self.assertEqual(caumanns('xxem'), 'xxem') |
||
| 835 | self.assertEqual(caumanns('xxer'), 'xxer') |
||
| 836 | # -nd is not removed from words shorter than six characters: |
||
| 837 | self.assertEqual(caumanns('xxxnd'), 'xxxnd') |
||
| 838 | |||
| 839 | |||
| 840 | class UEALiteTestCases(unittest.TestCase): |
||
| 841 | """Test UEA-lite functions. |
||
| 842 | |||
| 843 | abydos.stemmer.uealite |
||
| 844 | """ |
||
| 845 | |||
| 846 | def test_uealite(self): |
||
| 847 | """Test abydos.stemmer.uealite.""" |
||
| 848 | # base case |
||
| 849 | self.assertEqual(uealite(''), '') |
||
| 850 | |||
| 851 | # test cases copied from Ruby port |
||
| 852 | # https://github.com/ealdent/uea-stemmer/blob/master/test/uea_stemmer_test.rb |
||
| 853 | # These are corrected to match the Java version's output. |
||
| 854 | # stem base words to just the base word |
||
| 855 | self.assertEqual(uealite('man'), 'man') |
||
| 856 | self.assertEqual(uealite('happiness'), 'happiness') |
||
| 857 | # stem theses as thesis but not bases as basis |
||
| 858 | self.assertEqual(uealite('theses'), 'thesis') |
||
| 859 | self.assertNotEqual(uealite('bases'), 'basis') |
||
| 860 | # stem preterite words ending in -ed without the -ed |
||
| 861 | self.assertEqual(uealite('ordained'), 'ordain') |
||
| 862 | self.assertEqual(uealite('killed'), 'kill') |
||
| 863 | self.assertEqual(uealite('liked'), 'lik') |
||
| 864 | self.assertEqual(uealite('helped'), 'help') |
||
| 865 | self.assertEqual(uealite('scarred'), 'scarre') |
||
| 866 | self.assertEqual(uealite('invited'), 'invit') |
||
| 867 | self.assertEqual(uealite('exited'), 'exit') |
||
| 868 | self.assertEqual(uealite('debited'), 'debit') |
||
| 869 | self.assertEqual(uealite('smited'), 'smit') |
||
| 870 | # stem progressive verbs and gerunds without the -ing |
||
| 871 | self.assertEqual(uealite('running'), 'run') |
||
| 872 | self.assertEqual(uealite('settings'), 'set') |
||
| 873 | self.assertEqual(uealite('timing'), 'time') |
||
| 874 | self.assertEqual(uealite('dying'), 'dy') |
||
| 875 | self.assertEqual(uealite('harping'), 'harp') |
||
| 876 | self.assertEqual(uealite('charring'), 'char') |
||
| 877 | # not stem false progressive verbs such as 'sing' |
||
| 878 | self.assertEqual(uealite('ring'), 'ring') |
||
| 879 | self.assertEqual(uealite('sing'), 'se') |
||
| 880 | self.assertEqual(uealite('bring'), 'br') |
||
| 881 | self.assertEqual(uealite('fling'), 'fle') |
||
| 882 | # stem various plural nouns and 3rd-pres verbs without the -s/-es |
||
| 883 | self.assertEqual(uealite('changes'), 'change') |
||
| 884 | self.assertEqual(uealite('deaths'), 'death') |
||
| 885 | self.assertEqual(uealite('shadows'), 'shadow') |
||
| 886 | self.assertEqual(uealite('flies'), 'fly') |
||
| 887 | self.assertEqual(uealite('things'), 'thing') |
||
| 888 | self.assertEqual(uealite('nothings'), 'nothing') |
||
| 889 | self.assertEqual(uealite('witches'), 'witch') |
||
| 890 | self.assertEqual(uealite('makes'), 'mak') |
||
| 891 | self.assertEqual(uealite('smokes'), 'smok') |
||
| 892 | self.assertEqual(uealite('does'), 'do') |
||
| 893 | # stem various words with -des suffix |
||
| 894 | self.assertEqual(uealite('abodes'), 'abod') |
||
| 895 | self.assertEqual(uealite('escapades'), 'escapad') |
||
| 896 | self.assertEqual(uealite('crusades'), 'crusad') |
||
| 897 | self.assertEqual(uealite('grades'), 'grad') |
||
| 898 | # stem various words with -res suffix |
||
| 899 | self.assertEqual(uealite('wires'), 'wir') |
||
| 900 | self.assertEqual(uealite('acres'), 'acr') |
||
| 901 | self.assertEqual(uealite('fires'), 'fir') |
||
| 902 | self.assertEqual(uealite('cares'), 'car') |
||
| 903 | # stem acronyms when pluralized otherwise they should be left alone |
||
| 904 | self.assertEqual(uealite('USA'), 'USA') |
||
| 905 | self.assertEqual(uealite('FLOSS'), 'FLOSS') |
||
| 906 | self.assertEqual(uealite('MREs'), 'MRE') |
||
| 907 | self.assertEqual(uealite('USAED'), 'USAED') |
||
| 908 | |||
| 909 | # test cases copied from Ruby port |
||
| 910 | # https://github.com/ealdent/uea-stemmer/blob/master/test/uea_stemmer_test.rb |
||
| 911 | # stem base words to just the base word |
||
| 912 | self.assertEqual(uealite('man', var='Adams'), 'man') |
||
| 913 | self.assertEqual(uealite('happiness', var='Adams'), 'happiness') |
||
| 914 | # stem theses as thesis but not bases as basis |
||
| 915 | self.assertEqual(uealite('theses', var='Adams'), 'thesis') |
||
| 916 | self.assertNotEqual(uealite('bases', var='Adams'), 'basis') |
||
| 917 | # stem preterite words ending in -ed without the -ed |
||
| 918 | self.assertEqual(uealite('ordained', var='Adams'), 'ordain') |
||
| 919 | self.assertEqual(uealite('killed', var='Adams'), 'kill') |
||
| 920 | self.assertEqual(uealite('liked', var='Adams'), 'like') |
||
| 921 | self.assertEqual(uealite('helped', var='Adams'), 'help') |
||
| 922 | # self.assertEqual(uealite('scarred', var='Adams'), 'scar') |
||
| 923 | self.assertEqual(uealite('invited', var='Adams'), 'invite') |
||
| 924 | self.assertEqual(uealite('exited', var='Adams'), 'exit') |
||
| 925 | self.assertEqual(uealite('debited', var='Adams'), 'debit') |
||
| 926 | self.assertEqual(uealite('smited', var='Adams'), 'smite') |
||
| 927 | # stem progressive verbs and gerunds without the -ing |
||
| 928 | self.assertEqual(uealite('running', var='Adams'), 'run') |
||
| 929 | self.assertEqual(uealite('settings', var='Adams'), 'set') |
||
| 930 | self.assertEqual(uealite('timing', var='Adams'), 'time') |
||
| 931 | self.assertEqual(uealite('dying', var='Adams'), 'die') |
||
| 932 | self.assertEqual(uealite('harping', var='Adams'), 'harp') |
||
| 933 | self.assertEqual(uealite('charring', var='Adams'), 'char') |
||
| 934 | # not stem false progressive verbs such as 'sing' |
||
| 935 | self.assertEqual(uealite('ring', var='Adams'), 'ring') |
||
| 936 | self.assertEqual(uealite('sing', var='Adams'), 'sing') |
||
| 937 | self.assertEqual(uealite('ring', var='Adams'), 'ring') |
||
| 938 | self.assertEqual(uealite('bring', var='Adams'), 'bring') |
||
| 939 | self.assertEqual(uealite('fling', var='Adams'), 'fling') |
||
| 940 | # stem various plural nouns and 3rd-pres verbs without the -s/-es |
||
| 941 | self.assertEqual(uealite('changes', var='Adams'), 'change') |
||
| 942 | self.assertEqual(uealite('deaths', var='Adams'), 'death') |
||
| 943 | self.assertEqual(uealite('shadows', var='Adams'), 'shadow') |
||
| 944 | self.assertEqual(uealite('flies', var='Adams'), 'fly') |
||
| 945 | self.assertEqual(uealite('things', var='Adams'), 'thing') |
||
| 946 | self.assertEqual(uealite('nothings', var='Adams'), 'nothing') |
||
| 947 | self.assertEqual(uealite('witches', var='Adams'), 'witch') |
||
| 948 | self.assertEqual(uealite('makes', var='Adams'), 'make') |
||
| 949 | self.assertEqual(uealite('smokes', var='Adams'), 'smoke') |
||
| 950 | self.assertEqual(uealite('does', var='Adams'), 'do') |
||
| 951 | # stem various words with -des suffix |
||
| 952 | self.assertEqual(uealite('abodes', var='Adams'), 'abode') |
||
| 953 | self.assertEqual(uealite('escapades', var='Adams'), 'escapade') |
||
| 954 | self.assertEqual(uealite('crusades', var='Adams'), 'crusade') |
||
| 955 | self.assertEqual(uealite('grades', var='Adams'), 'grade') |
||
| 956 | # stem various words with -res suffix |
||
| 957 | self.assertEqual(uealite('wires', var='Adams'), 'wire') |
||
| 958 | self.assertEqual(uealite('acres', var='Adams'), 'acre') |
||
| 959 | self.assertEqual(uealite('fires', var='Adams'), 'fire') |
||
| 960 | self.assertEqual(uealite('cares', var='Adams'), 'care') |
||
| 961 | # stem acronyms when pluralized otherwise they should be left alone |
||
| 962 | self.assertEqual(uealite('USA', var='Adams'), 'USA') |
||
| 963 | self.assertEqual(uealite('FLOSS', var='Adams'), 'FLOSS') |
||
| 964 | self.assertEqual(uealite('MREs', var='Adams'), 'MRE') |
||
| 965 | self.assertEqual(uealite('USAED', var='Adams'), 'USAED') |
||
| 966 | |||
| 967 | def test_uealite_wsj_set(self): |
||
| 968 | """Test abydos.stemmer.uealite (WSJ testset).""" |
||
| 969 | with open(TESTDIR + '/corpora/uea-lite_wsj.csv') as wsj_testset: |
||
| 970 | for wsj_line in wsj_testset: |
||
| 971 | (word, uea, rule) = wsj_line.strip().split(',') |
||
| 972 | self.assertEqual(uealite(word, return_rule_no=True), |
||
| 973 | (uea, float(rule))) |
||
| 974 | |||
| 975 | |||
| 976 | class PaiceHuskTestCases(unittest.TestCase): |
||
| 977 | """Test Paice-Husk functions. |
||
| 978 | |||
| 979 | abydos.stemmer.paice_husk |
||
| 980 | """ |
||
| 981 | |||
| 982 | def test_paice_husk(self): |
||
| 983 | """Test abydos.stemmer.paice_husk.""" |
||
| 984 | # base case |
||
| 985 | self.assertEqual(paice_husk(''), '') |
||
| 986 | |||
| 987 | # cases copied from |
||
| 988 | # https://doi.org/10.1145/101306.101310 |
||
| 989 | self.assertEqual(paice_husk('maximum'), 'maxim') |
||
| 990 | self.assertEqual(paice_husk('presumably'), 'presum') |
||
| 991 | self.assertEqual(paice_husk('multiply'), 'multiply') |
||
| 992 | self.assertEqual(paice_husk('provision'), 'provid') |
||
| 993 | self.assertEqual(paice_husk('owed'), 'ow') |
||
| 994 | self.assertEqual(paice_husk('owing'), 'ow') |
||
| 995 | self.assertEqual(paice_husk('ear'), 'ear') |
||
| 996 | self.assertEqual(paice_husk('saying'), 'say') |
||
| 997 | self.assertEqual(paice_husk('crying'), 'cry') |
||
| 998 | self.assertEqual(paice_husk('string'), 'string') |
||
| 999 | self.assertEqual(paice_husk('meant'), 'meant') |
||
| 1000 | self.assertEqual(paice_husk('cement'), 'cem') |
||
| 1001 | |||
| 1002 | def test_paice_husk_hopper_set(self): |
||
| 1003 | """Test abydos.stemmer.paice_husk (Hopper262 testset). |
||
| 1004 | |||
| 1005 | Source: |
||
| 1006 | https://raw.githubusercontent.com/Hopper262/paice-husk-stemmer/master/wordlist.txt |
||
| 1007 | |||
| 1008 | The only correction made from stemmed values in the Hopper262 set/ |
||
| 1009 | implementations were: |
||
| 1010 | - ymca : ymc -> ymca |
||
| 1011 | - yttrium : yttr -> yttri |
||
| 1012 | - ywca : ywc -> ywca |
||
| 1013 | The Pascal reference implementation does not consider 'y' in initial |
||
| 1014 | position to be a vowel. |
||
| 1015 | """ |
||
| 1016 | with open(TESTDIR + '/corpora/paicehusk.csv') as hopper_testset: |
||
| 1017 | for hopper_line in hopper_testset: |
||
| 1018 | (word, stem) = hopper_line.strip().split(',') |
||
| 1019 | self.assertEqual(paice_husk(word), stem) |
||
| 1020 | |||
| 1021 | |||
| 1022 | if __name__ == '__main__': |
||
| 1023 | unittest.main() |
||
| 1024 |
Loading history...