1
|
|
|
import copy |
2
|
|
|
import os |
3
|
|
|
import warnings |
4
|
|
|
|
5
|
|
|
import matplotlib.pylab as plt |
6
|
|
|
import numpy as np |
7
|
|
|
import pandas as pd |
8
|
|
|
import seaborn as sns |
9
|
|
|
from gensim.models.keyedvectors import KeyedVectors |
10
|
|
|
from pkg_resources import resource_filename |
11
|
|
|
from sklearn.decomposition import PCA |
12
|
|
|
from sklearn.svm import LinearSVC |
13
|
|
|
from tqdm import tqdm |
14
|
|
|
|
15
|
|
|
from ..consts import RANDOM_STATE |
16
|
|
|
from .utils import ( |
17
|
|
|
cosine_similarity, normalize, project_reject_vector, project_vector, |
18
|
|
|
reject_vector, update_word_vector, |
19
|
|
|
) |
20
|
|
|
|
21
|
|
|
|
22
|
|
|
DIRECTION_METHODS = ['single', 'sum', 'pca'] |
23
|
|
|
DEBIAS_METHODS = ['neutralize', 'hard', 'soft'] |
24
|
|
|
FIRST_PC_THRESHOLD = 0.5 |
25
|
|
|
MAX_NON_SPECIFIC_EXAMPLES = 1000 |
26
|
|
|
|
27
|
|
|
|
28
|
|
|
class BiasWordsEmbedding: |
29
|
|
|
|
30
|
|
|
def __init__(self, model, only_lower=True): |
31
|
|
|
if not isinstance(model, KeyedVectors): |
32
|
|
|
raise TypeError('model should be of type KeyedVectors, not {}' |
33
|
|
|
.format(type(model))) |
34
|
|
|
|
35
|
|
|
self.model = model |
36
|
|
|
|
37
|
|
|
# TODO: write unitest for when it is False |
38
|
|
|
self.only_lower = only_lower |
39
|
|
|
|
40
|
|
|
self.direction = None |
41
|
|
|
self.positive_end = None |
42
|
|
|
self.negative_end = None |
43
|
|
|
|
44
|
|
|
def __copy__(self): |
45
|
|
|
bias_words_embedding = self.__class__(self.model) |
46
|
|
|
bias_words_embedding.direction = copy.deepcopy(self.direction) |
47
|
|
|
bias_words_embedding.positive_end = copy.deepcopy(self.positive_end) |
48
|
|
|
bias_words_embedding.negative_end = copy.deepcopy(self.negative_end) |
49
|
|
|
return bias_words_embedding |
50
|
|
|
|
51
|
|
|
def __deepcopy__(self, memo): |
52
|
|
|
bias_words_embedding = copy.copy(self) |
53
|
|
|
bias_words_embedding.model = copy.deepcopy(bias_words_embedding.model) |
54
|
|
|
return bias_words_embedding |
55
|
|
|
|
56
|
|
|
def __getitem__(self, key): |
57
|
|
|
return self.model[key] |
58
|
|
|
|
59
|
|
|
def __contains__(self, item): |
60
|
|
|
return item in self.model |
61
|
|
|
|
62
|
|
|
def _filter_words_by_model(self, words): |
63
|
|
|
return [word for word in words if word in self] |
64
|
|
|
|
65
|
|
|
def _is_direction_identified(self): |
66
|
|
|
if self.direction is None: |
67
|
|
|
raise RuntimeError('The direction was not identified' |
68
|
|
|
' for this {} instance' |
69
|
|
|
.format(self.__class__.__name__)) |
70
|
|
|
|
71
|
|
|
# There is a mistake in the article |
72
|
|
|
# it is written (section 5.1): |
73
|
|
|
# "To identify the gender subspace, we took the ten gender pair difference |
74
|
|
|
# vectors and computed its principal components (PCs)" |
75
|
|
|
# however in the source code: |
76
|
|
|
# https://github.com/tolga-b/debiaswe/blob/10277b23e187ee4bd2b6872b507163ef4198686b/debiaswe/we.py#L235-L245 |
77
|
|
|
def _identify_subspace_by_pca(self, definitional_pairs, n_components): |
78
|
|
|
matrix = [] |
79
|
|
|
|
80
|
|
|
for word1, word2 in definitional_pairs: |
81
|
|
|
vector1 = normalize(self[word1]) |
82
|
|
|
vector2 = normalize(self[word2]) |
83
|
|
|
|
84
|
|
|
center = (vector1 + vector2) / 2 |
85
|
|
|
|
86
|
|
|
matrix.append(vector1 - center) |
87
|
|
|
matrix.append(vector2 - center) |
88
|
|
|
|
89
|
|
|
pca = PCA(n_components=n_components) |
90
|
|
|
pca.fit(matrix) |
91
|
|
|
|
92
|
|
|
return pca |
93
|
|
|
|
94
|
|
|
# TODO: add the SVD method from section 6 step 1 |
95
|
|
|
# It seems there is a mistake there, I think it is the same as PCA |
96
|
|
|
# just with repleacing it with SVD |
97
|
|
|
def _identify_direction(self, positive_end, negative_end, |
98
|
|
|
definitional, method='pca'): |
99
|
|
|
if method not in DIRECTION_METHODS: |
100
|
|
|
raise ValueError('method should be one of {}, {} was given'.format( |
101
|
|
|
DIRECTION_METHODS, method)) |
102
|
|
|
|
103
|
|
|
if positive_end == negative_end: |
104
|
|
|
raise ValueError('positive_end and negative_end' |
105
|
|
|
'should be different, and not the same "{}"' |
106
|
|
|
.format(positive_end)) |
107
|
|
|
|
108
|
|
|
direction = None |
109
|
|
|
|
110
|
|
|
if method == 'single': |
111
|
|
|
direction = normalize(normalize(self[definitional[0]]) |
112
|
|
|
- normalize(self[definitional[1]])) |
113
|
|
|
|
114
|
|
|
elif method == 'sum': |
115
|
|
|
groups = list(zip(*definitional)) |
116
|
|
|
|
117
|
|
|
group1_sum_vector = np.sum([self[word] |
118
|
|
|
for word in groups[0]], axis=0) |
119
|
|
|
group2_sum_vector = np.sum([self[word] |
120
|
|
|
for word in groups[1]], axis=0) |
121
|
|
|
|
122
|
|
|
diff_vector = (normalize(group1_sum_vector) |
123
|
|
|
- normalize(group2_sum_vector)) |
124
|
|
|
|
125
|
|
|
direction = normalize(diff_vector) |
126
|
|
|
|
127
|
|
|
elif method == 'pca': |
128
|
|
|
pca = self._identify_subspace_by_pca(definitional, 1) |
129
|
|
|
if pca.explained_variance_ratio_[0] < FIRST_PC_THRESHOLD: |
130
|
|
|
raise RuntimeError('The Explained variance' |
131
|
|
|
'of the first principal component should be' |
132
|
|
|
'at least {}, but it is {}' |
133
|
|
|
.format(FIRST_PC_THRESHOLD, |
134
|
|
|
pca.explained_variance_ratio_[0])) |
135
|
|
|
direction = pca.components_[0] |
136
|
|
|
|
137
|
|
|
# if direction is oposite (e.g. we cannot control |
138
|
|
|
# what the PCA will return) |
139
|
|
|
ends_diff_projection = cosine_similarity((self[positive_end] |
140
|
|
|
- self[negative_end]), |
141
|
|
|
direction) |
142
|
|
|
if ends_diff_projection < 0: |
143
|
|
|
direction = -direction # pylint: disable=invalid-unary-operand-type |
144
|
|
|
|
145
|
|
|
self.direction = direction |
146
|
|
|
self.positive_end = positive_end |
147
|
|
|
self.negative_end = negative_end |
148
|
|
|
|
149
|
|
|
def project_on_direction(self, word): |
150
|
|
|
self._is_direction_identified() |
151
|
|
|
|
152
|
|
|
vector = self[word] |
153
|
|
|
projection_score = self.model.cosine_similarities(self.direction, |
154
|
|
|
[vector])[0] |
155
|
|
|
return projection_score |
156
|
|
|
|
157
|
|
|
def _calc_projection_scores(self, words): |
158
|
|
|
self._is_direction_identified() |
159
|
|
|
|
160
|
|
|
df = pd.DataFrame({'word': words}) |
161
|
|
|
|
162
|
|
|
# TODO: maybe using cosine_similarities on all the vectors? |
163
|
|
|
# it might be faster |
164
|
|
|
df['projection'] = df['word'].apply(self.project_on_direction) |
165
|
|
|
df = df.sort_values('projection', ascending=False) |
166
|
|
|
|
167
|
|
|
return df |
168
|
|
|
|
169
|
|
|
def plot_projection_scores(self, words, |
170
|
|
|
ax=None, axis_projection_step=None): |
171
|
|
|
self._is_direction_identified() |
172
|
|
|
|
173
|
|
|
projections_df = self._calc_projection_scores(words) |
174
|
|
|
projections_df['projection'] = projections_df['projection'].round(2) |
175
|
|
|
|
176
|
|
|
if ax is None: |
177
|
|
|
_, ax = plt.subplots(1) |
178
|
|
|
|
179
|
|
|
if axis_projection_step is None: |
180
|
|
|
axis_projection_step = 0.1 |
181
|
|
|
|
182
|
|
|
cmap = plt.get_cmap('RdBu') |
183
|
|
|
projections_df['color'] = ((projections_df['projection'] + 0.5) |
184
|
|
|
.apply(cmap)) |
185
|
|
|
|
186
|
|
|
most_extream_projection = (projections_df['projection'] |
187
|
|
|
.abs() |
188
|
|
|
.max() |
189
|
|
|
.round(1)) |
190
|
|
|
|
191
|
|
|
sns.barplot(x='projection', y='word', data=projections_df, |
192
|
|
|
palette=projections_df['color']) |
193
|
|
|
|
194
|
|
|
plt.xticks(np.arange(-most_extream_projection, most_extream_projection, |
195
|
|
|
axis_projection_step)) |
196
|
|
|
plt.title('← {} {} {} →'.format(self.negative_end, |
197
|
|
|
' ' * 20, |
198
|
|
|
self.positive_end)) |
199
|
|
|
|
200
|
|
|
plt.xlabel('Direction Projection') |
201
|
|
|
plt.ylabel('Words') |
202
|
|
|
|
203
|
|
|
def calc_direct_bias(self, neutral_words, c=None): |
204
|
|
|
if c is None: |
205
|
|
|
c = 1 |
206
|
|
|
|
207
|
|
|
projections = self._calc_projection_scores(neutral_words)['projection'] |
208
|
|
|
direct_bias_terms = np.abs(projections) ** c |
209
|
|
|
direct_bias = direct_bias_terms.sum() / len(neutral_words) |
210
|
|
|
|
211
|
|
|
return direct_bias |
212
|
|
|
|
213
|
|
|
def calc_indirect_bias(self, word1, word2): |
214
|
|
|
"""Also known in the article as PairBias.""" |
215
|
|
|
self._is_direction_identified() |
216
|
|
|
|
217
|
|
|
vector1 = normalize(self[word1]) |
218
|
|
|
vector2 = normalize(self[word2]) |
219
|
|
|
|
220
|
|
|
perpendicular_vector1 = reject_vector(vector1, self.direction) |
221
|
|
|
perpendicular_vector2 = reject_vector(vector2, self.direction) |
222
|
|
|
|
223
|
|
|
inner_product = vector1 @ vector2 |
224
|
|
|
perpendicular_similarity = cosine_similarity(perpendicular_vector1, |
225
|
|
|
perpendicular_vector2) |
226
|
|
|
|
227
|
|
|
indirect_bias = ((inner_product - perpendicular_similarity) |
228
|
|
|
/ inner_product) |
229
|
|
|
return indirect_bias |
230
|
|
|
|
231
|
|
|
def _extract_neutral_words(self, specific_words): |
232
|
|
|
extended_specific_words = set() |
233
|
|
|
|
234
|
|
|
# because or specific_full data was trained on partial words embedding |
235
|
|
|
for word in specific_words: |
236
|
|
|
extended_specific_words.add(word) |
237
|
|
|
extended_specific_words.add(word.lower()) |
238
|
|
|
extended_specific_words.add(word.upper()) |
239
|
|
|
extended_specific_words.add(word.title()) |
240
|
|
|
|
241
|
|
|
neutral_words = [word for word in self.model.vocab |
242
|
|
|
if word not in extended_specific_words] |
243
|
|
|
|
244
|
|
|
return neutral_words |
245
|
|
|
|
246
|
|
|
def _neutralize(self, neutral_words, verbose=False): |
247
|
|
|
self._is_direction_identified() |
248
|
|
|
|
249
|
|
|
if verbose: |
250
|
|
|
neutral_words_iter = tqdm(neutral_words) |
251
|
|
|
else: |
252
|
|
|
neutral_words_iter = iter(neutral_words) |
253
|
|
|
|
254
|
|
|
for word in neutral_words_iter: |
255
|
|
|
neutralized_vector = reject_vector(self[word], |
256
|
|
|
self.direction) |
257
|
|
|
update_word_vector(self.model, word, neutralized_vector) |
258
|
|
|
|
259
|
|
|
self.model.init_sims(replace=True) |
260
|
|
|
|
261
|
|
|
def _equalize(self, equality_sets): |
262
|
|
|
for equality_set_words in equality_sets: |
263
|
|
|
equality_set_vectors = [normalize(self[word]) |
264
|
|
|
for word in equality_set_words] |
265
|
|
|
center = np.mean(equality_set_vectors, axis=0) |
266
|
|
|
(projected_center, |
267
|
|
|
rejected_center) = project_reject_vector(center, |
268
|
|
|
self.direction) |
269
|
|
|
|
270
|
|
|
for word, vector in zip(equality_set_words, equality_set_vectors): |
271
|
|
|
projected_vector = project_vector(vector, self.direction) |
272
|
|
|
|
273
|
|
|
projected_part = normalize(projected_vector - projected_center) |
274
|
|
|
scaling = np.sqrt(1 - np.linalg.norm(rejected_center)**2) |
275
|
|
|
|
276
|
|
|
# TODO - in the code it is different - why? |
277
|
|
|
# equalized_vector = rejected_center + scaling * self.direction |
278
|
|
|
# https://github.com/tolga-b/debiaswe/blob/10277b23e187ee4bd2b6872b507163ef4198686b/debiaswe/debias.py#L36-L37 |
279
|
|
|
equalized_vector = rejected_center + scaling * projected_part |
280
|
|
|
|
281
|
|
|
update_word_vector(self.model, word, equalized_vector) |
282
|
|
|
|
283
|
|
|
self.model.init_sims(replace=True) |
284
|
|
|
|
285
|
|
|
def debias(self, method='hard', neutral_words=None, equality_sets=None, |
286
|
|
|
inplace=True, verbose=False): |
287
|
|
|
# pylint: disable=W0212 |
288
|
|
|
if inplace: |
289
|
|
|
bias_words_embedding = self |
290
|
|
|
else: |
291
|
|
|
bias_words_embedding = copy.deepcopy(self) |
292
|
|
|
|
293
|
|
|
if method not in DEBIAS_METHODS: |
294
|
|
|
raise ValueError('method should be one of {}, {} was given'.format( |
295
|
|
|
DEBIAS_METHODS, method)) |
296
|
|
|
|
297
|
|
|
if method in ['hard', 'neutralize']: |
298
|
|
|
if verbose: |
299
|
|
|
print('Neutralize...') |
300
|
|
|
bias_words_embedding._neutralize(neutral_words, verbose) |
301
|
|
|
|
302
|
|
|
if method == 'hard': |
303
|
|
|
if verbose: |
304
|
|
|
print('Equalize...') |
305
|
|
|
bias_words_embedding._equalize(equality_sets) |
306
|
|
|
|
307
|
|
|
if inplace: |
308
|
|
|
return None |
309
|
|
|
else: |
310
|
|
|
return bias_words_embedding |
311
|
|
|
|
312
|
|
|
def evaluate_words_embedding(self, verbose=False): |
313
|
|
|
with warnings.catch_warnings(): |
314
|
|
|
warnings.simplefilter('ignore', category=FutureWarning) |
315
|
|
|
|
316
|
|
|
if verbose: |
317
|
|
|
print('Evaluate word pairs...') |
318
|
|
|
word_pairs_path = resource_filename(__name__, |
319
|
|
|
os.path.join('data', |
320
|
|
|
'evaluation', |
321
|
|
|
'wordsim353.tsv')) |
322
|
|
|
word_paris_result = self.model.evaluate_word_pairs(word_pairs_path) |
323
|
|
|
|
324
|
|
|
if verbose: |
325
|
|
|
print('Evaluate analogies...') |
326
|
|
|
analogies_path = resource_filename(__name__, |
327
|
|
|
os.path.join('data', |
328
|
|
|
'evaluation', |
329
|
|
|
'questions-words.txt')) # pylint: disable=C0301 |
330
|
|
|
analogies_result = self.model.evaluate_word_analogies(analogies_path) # pylint: disable=C0301 |
331
|
|
|
|
332
|
|
|
if verbose: |
333
|
|
|
print() |
334
|
|
|
print('From Gensim') |
335
|
|
|
print() |
336
|
|
|
print('-' * 30) |
337
|
|
|
print() |
338
|
|
|
print('Word Pairs Result - WordSimilarity-353:') |
339
|
|
|
print('~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~') |
340
|
|
|
print('Pearson correlation coefficient:', word_paris_result[0]) |
341
|
|
|
print('Spearman rank-order correlation coefficient' |
342
|
|
|
'between the similarities from the dataset' |
343
|
|
|
'and the similarities produced by the model itself:', |
344
|
|
|
word_paris_result[1]) |
345
|
|
|
print('Ratio of pairs with unknown words:', word_paris_result[2]) |
346
|
|
|
print() |
347
|
|
|
print('-' * 30) |
348
|
|
|
print() |
349
|
|
|
print('Analogies Result') |
350
|
|
|
print('~~~~~~~~~~~~~~~~') |
351
|
|
|
print('Overall evaluation score:', analogies_result[0]) |
352
|
|
|
|
353
|
|
|
def learn_full_specific_words(self, seed_specific_words, |
354
|
|
|
max_non_specific_examples=None, debug=None): |
355
|
|
|
|
356
|
|
|
if debug is None: |
357
|
|
|
debug = False |
358
|
|
|
|
359
|
|
|
if max_non_specific_examples is None: |
360
|
|
|
max_non_specific_examples = MAX_NON_SPECIFIC_EXAMPLES |
361
|
|
|
|
362
|
|
|
data = [] |
363
|
|
|
non_specific_example_count = 0 |
364
|
|
|
|
365
|
|
|
for word in self.model.vocab: |
366
|
|
|
is_specific = word in seed_specific_words |
367
|
|
|
|
368
|
|
|
if not is_specific: |
369
|
|
|
non_specific_example_count += 1 |
370
|
|
|
if non_specific_example_count <= max_non_specific_examples: |
371
|
|
|
data.append((self[word], is_specific)) |
372
|
|
|
else: |
373
|
|
|
data.append((self[word], is_specific)) |
374
|
|
|
|
375
|
|
|
np.random.seed(RANDOM_STATE) |
376
|
|
|
np.random.shuffle(data) |
377
|
|
|
|
378
|
|
|
X, y = zip(*data) |
379
|
|
|
|
380
|
|
|
X = np.array(X) |
381
|
|
|
X /= np.linalg.norm(X, axis=1)[:, None] |
382
|
|
|
|
383
|
|
|
y = np.array(y).astype('int') |
384
|
|
|
|
385
|
|
|
clf = LinearSVC(C=1, class_weight='balanced', |
386
|
|
|
random_state=RANDOM_STATE) |
387
|
|
|
|
388
|
|
|
clf.fit(X, y) |
389
|
|
|
|
390
|
|
|
full_specific_words = [] |
391
|
|
|
for word in self.model.vocab: |
392
|
|
|
vector = [normalize(self[word])] |
393
|
|
|
if clf.predict(vector): |
394
|
|
|
full_specific_words.append(word) |
395
|
|
|
|
396
|
|
|
if not debug: |
397
|
|
|
return full_specific_words, clf |
398
|
|
|
|
399
|
|
|
return full_specific_words, clf, X, y |
400
|
|
|
|