|
1
|
|
|
"""Annif backend using Yake keyword extraction""" |
|
2
|
|
|
# TODO Mention GPLv3 license also here? |
|
3
|
|
|
|
|
4
|
|
|
import yake |
|
5
|
|
|
import os.path |
|
6
|
|
|
import re |
|
7
|
|
|
from collections import defaultdict |
|
8
|
|
|
from rdflib.namespace import SKOS, RDF, OWL |
|
9
|
|
|
import rdflib |
|
10
|
|
|
import annif.util |
|
11
|
|
|
from . import backend |
|
12
|
|
|
from annif.suggestion import SubjectSuggestion, ListSuggestionResult |
|
13
|
|
|
from annif.exception import ConfigurationException |
|
14
|
|
|
|
|
15
|
|
|
|
|
16
|
|
|
class YakeBackend(backend.AnnifBackend): |
|
17
|
|
|
"""Yake based backend for Annif""" |
|
18
|
|
|
name = "yake" |
|
19
|
|
|
needs_subject_index = False |
|
20
|
|
|
|
|
21
|
|
|
# defaults for uninitialized instances |
|
22
|
|
|
_index = None |
|
23
|
|
|
_graph = None |
|
24
|
|
|
INDEX_FILE = 'yake-index' |
|
25
|
|
|
|
|
26
|
|
|
DEFAULT_PARAMETERS = { |
|
27
|
|
|
'max_ngram_size': 4, |
|
28
|
|
|
'deduplication_threshold': 0.9, |
|
29
|
|
|
'deduplication_algo': 'levs', |
|
30
|
|
|
'window_size': 1, |
|
31
|
|
|
'num_keywords': 100, |
|
32
|
|
|
'features': None, |
|
33
|
|
|
'default_label_types': ['pref', 'alt'], |
|
34
|
|
|
'remove_specifiers': False |
|
35
|
|
|
} |
|
36
|
|
|
|
|
37
|
|
|
def default_params(self): |
|
38
|
|
|
params = backend.AnnifBackend.DEFAULT_PARAMETERS.copy() |
|
39
|
|
|
params.update(self.DEFAULT_PARAMETERS) |
|
40
|
|
|
return params |
|
41
|
|
|
|
|
42
|
|
|
@property |
|
43
|
|
|
def is_trained(self): |
|
44
|
|
|
return True |
|
45
|
|
|
|
|
46
|
|
|
@property |
|
47
|
|
|
def label_types(self): |
|
48
|
|
|
mapping = {'pref': SKOS.prefLabel, |
|
49
|
|
|
'alt': SKOS.altLabel, |
|
50
|
|
|
'hidden': SKOS.hiddenLabel} |
|
51
|
|
|
if 'label_types' in self.params: |
|
52
|
|
|
lt_entries = self.params['label_types'].split(',') |
|
53
|
|
|
try: |
|
54
|
|
|
return [mapping[lt.strip()] for lt in lt_entries] |
|
55
|
|
|
except KeyError as err: |
|
56
|
|
|
raise ConfigurationException( |
|
57
|
|
|
f'invalid label type {err}', backend_id=self.backend_id) |
|
58
|
|
|
else: |
|
59
|
|
|
return [mapping[lt] for lt in self.params['default_label_types']] |
|
60
|
|
|
|
|
61
|
|
|
def initialize(self): |
|
62
|
|
|
self._initialize_index() |
|
63
|
|
|
self._kw_extractor = yake.KeywordExtractor( |
|
64
|
|
|
lan=self.params['language'], |
|
65
|
|
|
n=self.params['max_ngram_size'], |
|
66
|
|
|
dedupLim=self.params['deduplication_threshold'], |
|
67
|
|
|
dedupFunc=self.params['deduplication_algo'], |
|
68
|
|
|
windowsSize=self.params['window_size'], |
|
69
|
|
|
top=self.params['num_keywords'], |
|
70
|
|
|
features=self.params['features']) |
|
71
|
|
|
|
|
72
|
|
|
def _initialize_index(self): |
|
73
|
|
|
if self._index is None: |
|
74
|
|
|
path = os.path.join(self.datadir, self.INDEX_FILE) |
|
75
|
|
|
if os.path.exists(path): |
|
76
|
|
|
self._index = self._load_index(path) |
|
77
|
|
|
self.info( |
|
78
|
|
|
f'Loaded index from {path} with {len(self._index)} labels') |
|
79
|
|
|
else: |
|
80
|
|
|
self.info('Creating index') |
|
81
|
|
|
self._create_index() |
|
82
|
|
|
self._save_index(path) |
|
83
|
|
|
self.info(f'Created index with {len(self._index)} labels') |
|
84
|
|
|
|
|
85
|
|
|
@property |
|
86
|
|
|
def graph(self): |
|
87
|
|
|
if self._graph is None: |
|
88
|
|
|
self.info('Loading graph') |
|
89
|
|
|
self._graph = self.project.vocab.as_graph() |
|
90
|
|
|
return self._graph |
|
91
|
|
|
|
|
92
|
|
|
def _create_index(self): |
|
93
|
|
|
# TODO Should index creation & saving be done on loadvoc command? |
|
94
|
|
|
# Or saving at all? It takes about 1 min to create the index |
|
95
|
|
|
index = defaultdict(set) |
|
96
|
|
|
for label_type in self.label_types: |
|
97
|
|
|
for concept in self.graph.subjects(RDF.type, SKOS.Concept): |
|
98
|
|
|
if (concept, OWL.deprecated, rdflib.Literal(True)) \ |
|
99
|
|
|
in self.graph: |
|
100
|
|
|
continue |
|
101
|
|
|
for label in self.graph.objects(concept, label_type): |
|
102
|
|
|
if not label.language == self.params['language']: |
|
103
|
|
|
continue |
|
104
|
|
|
uri = str(concept) |
|
105
|
|
|
label = str(label) |
|
106
|
|
|
if annif.util.boolean(self.params['remove_specifiers']): |
|
107
|
|
|
label = re.sub(r' \(.*\)', '', label) |
|
108
|
|
|
lemmatized_label = self._lemmatize_phrase(label) |
|
109
|
|
|
lemmatized_label = self._sort_phrase(lemmatized_label) |
|
110
|
|
|
index[lemmatized_label].add(uri) |
|
111
|
|
|
index.pop('', None) # Remove possible empty string entry |
|
112
|
|
|
self._index = dict(index) |
|
113
|
|
|
|
|
114
|
|
|
def _save_index(self, path): |
|
115
|
|
|
with open(path, 'w', encoding='utf-8') as indexfile: |
|
116
|
|
|
for label, uris in self._index.items(): |
|
117
|
|
|
line = label + '\t' + ' '.join(uris) |
|
118
|
|
|
print(line, file=indexfile) |
|
119
|
|
|
|
|
120
|
|
|
def _load_index(self, path): |
|
121
|
|
|
index = dict() |
|
122
|
|
|
with open(path, 'r', encoding='utf-8') as indexfile: |
|
123
|
|
|
for line in indexfile: |
|
124
|
|
|
label, uris = line.strip().split('\t') |
|
125
|
|
|
index[label] = uris.split() |
|
126
|
|
|
return index |
|
127
|
|
|
|
|
128
|
|
|
def _sort_phrase(self, phrase): |
|
129
|
|
|
words = phrase.split() |
|
130
|
|
|
return ' '.join(sorted(words)) |
|
131
|
|
|
|
|
132
|
|
|
def _lemmatize_phrase(self, phrase): |
|
133
|
|
|
normalized = [] |
|
134
|
|
|
for word in phrase.split(): |
|
135
|
|
|
normalized.append( |
|
136
|
|
|
self.project.analyzer.normalize_word(word).lower()) |
|
137
|
|
|
return ' '.join(normalized) |
|
138
|
|
|
|
|
139
|
|
|
def _keyphrases2suggestions(self, keyphrases): |
|
140
|
|
|
suggestions = [] |
|
141
|
|
|
not_matched = [] |
|
142
|
|
|
for kp, score in keyphrases: |
|
143
|
|
|
uris = self._keyphrase2uris(kp) |
|
144
|
|
|
for uri in uris: |
|
145
|
|
|
suggestions.append( |
|
146
|
|
|
(uri, self._transform_score(score))) |
|
147
|
|
|
if not uris: |
|
148
|
|
|
not_matched.append((kp, self._transform_score(score))) |
|
149
|
|
|
# Remove duplicate uris, conflating the scores |
|
150
|
|
|
suggestions = self._combine_suggestions(suggestions) |
|
151
|
|
|
self.debug('Keyphrases not matched:\n' + '\t'.join( |
|
152
|
|
|
[kp[0] + ' ' + str(kp[1]) for kp |
|
153
|
|
|
in sorted(not_matched, reverse=True, key=lambda kp: kp[1])])) |
|
154
|
|
|
return suggestions |
|
155
|
|
|
|
|
156
|
|
|
def _keyphrase2uris(self, keyphrase): |
|
157
|
|
|
keyphrase = self._lemmatize_phrase(keyphrase) |
|
158
|
|
|
keyphrase = self._sort_phrase(keyphrase) |
|
159
|
|
|
return self._index.get(keyphrase, []) |
|
160
|
|
|
|
|
161
|
|
|
def _transform_score(self, score): |
|
162
|
|
|
if score < 0: |
|
163
|
|
|
self.debug(f'Replacing negative YAKE score {score} with zero') |
|
164
|
|
|
return 1.0 |
|
165
|
|
|
return 1.0 / (score + 1) |
|
166
|
|
|
|
|
167
|
|
|
def _combine_suggestions(self, suggestions): |
|
168
|
|
|
combined_suggestions = {} |
|
169
|
|
|
for uri, score in suggestions: |
|
170
|
|
|
if uri not in combined_suggestions: |
|
171
|
|
|
combined_suggestions[uri] = score |
|
172
|
|
|
else: |
|
173
|
|
|
old_score = combined_suggestions[uri] |
|
174
|
|
|
combined_suggestions[uri] = self._conflate_scores( |
|
175
|
|
|
score, old_score) |
|
176
|
|
|
return list(combined_suggestions.items()) |
|
177
|
|
|
|
|
178
|
|
|
def _conflate_scores(self, score1, score2): |
|
179
|
|
|
return score1 * score2 / (score1 * score2 + (1-score1) * (1-score2)) |
|
180
|
|
|
|
|
181
|
|
|
def _suggest(self, text, params): |
|
182
|
|
|
self.debug( |
|
183
|
|
|
f'Suggesting subjects for text "{text[:20]}..." (len={len(text)})') |
|
184
|
|
|
limit = int(params['limit']) |
|
185
|
|
|
|
|
186
|
|
|
keyphrases = self._kw_extractor.extract_keywords(text) |
|
187
|
|
|
suggestions = self._keyphrases2suggestions(keyphrases) |
|
188
|
|
|
|
|
189
|
|
|
subject_suggestions = [SubjectSuggestion( |
|
190
|
|
|
uri=uri, |
|
191
|
|
|
label=None, |
|
192
|
|
|
notation=None, |
|
193
|
|
|
score=score) |
|
194
|
|
|
for uri, score in suggestions[:limit] if score > 0.0] |
|
195
|
|
|
return ListSuggestionResult.create_from_index(subject_suggestions, |
|
196
|
|
|
self.project.subjects) |
|
197
|
|
|
|
NatLibFi /
Annif
