Passed
Push — master ( a15aca...2d6850 )
by Shlomi
01:44
created

BiasWordsEmbedding._filter_words_by_model()   A

Complexity

Conditions 1

Size

Total Lines 2
Code Lines 2

Duplication

Lines 0
Ratio 0 %

Importance

Changes 0
Metric Value
cc 1
eloc 2
nop 2
dl 0
loc 2
rs 10
c 0
b 0
f 0
1
import copy
2
import os
3
import warnings
4
5
import matplotlib.pylab as plt
6
import numpy as np
7
import pandas as pd
8
import seaborn as sns
9
from gensim.models.keyedvectors import KeyedVectors
10
from pkg_resources import resource_filename
11
from sklearn.decomposition import PCA
12
from sklearn.svm import LinearSVC
13
from tqdm import tqdm
14
15
from ..consts import RANDOM_STATE
16
from .utils import (
17
    cosine_similarity, normalize, project_reject_vector, project_vector,
18
    reject_vector, update_word_vector,
19
)
20
21
22
DIRECTION_METHODS = ['single', 'sum', 'pca']
23
DEBIAS_METHODS = ['neutralize', 'hard', 'soft']
24
FIRST_PC_THRESHOLD = 0.5
25
MAX_NON_SPECIFIC_EXAMPLES = 1000
26
27
28
class BiasWordsEmbedding:
29
30
    def __init__(self, model, only_lower=True):
31
        if not isinstance(model, KeyedVectors):
32
            raise TypeError('model should be of type KeyedVectors, not {}'
33
                            .format(type(model)))
34
35
        self.model = model
36
37
        # TODO: write unitest for when it is False
38
        self.only_lower = only_lower
39
40
        self.direction = None
41
        self.positive_end = None
42
        self.negative_end = None
43
44
    def __copy__(self):
45
        bias_words_embedding = self.__class__(self.model)
46
        bias_words_embedding.direction = copy.deepcopy(self.direction)
47
        bias_words_embedding.positive_end = copy.deepcopy(self.positive_end)
48
        bias_words_embedding.negative_end = copy.deepcopy(self.negative_end)
49
        return bias_words_embedding
50
51
    def __deepcopy__(self, memo):
52
        bias_words_embedding = copy.copy(self)
53
        bias_words_embedding.model = copy.deepcopy(bias_words_embedding.model)
54
        return bias_words_embedding
55
56
    def __getitem__(self, key):
57
        return self.model[key]
58
59
    def __contains__(self, item):
60
        return item in self.model
61
62
    def _filter_words_by_model(self, words):
63
        return [word for word in words if word in self]
64
65
    def _is_direction_identified(self):
66
        if self.direction is None:
67
            raise RuntimeError('The direction was not identified'
68
                               ' for this {} instance'
69
                               .format(self.__class__.__name__))
70
71
    # There is a mistake in the article
72
    # it is written (section 5.1):
73
    # "To identify the gender subspace, we took the ten gender pair difference
74
    # vectors and computed its principal components (PCs)"
75
    # however in the source code:
76
    # https://github.com/tolga-b/debiaswe/blob/10277b23e187ee4bd2b6872b507163ef4198686b/debiaswe/we.py#L235-L245
77
    def _identify_subspace_by_pca(self, definitional_pairs, n_components):
78
        matrix = []
79
80
        for word1, word2 in definitional_pairs:
81
            vector1 = normalize(self[word1])
82
            vector2 = normalize(self[word2])
83
84
            center = (vector1 + vector2) / 2
85
86
            matrix.append(vector1 - center)
87
            matrix.append(vector2 - center)
88
89
        pca = PCA(n_components=n_components)
90
        pca.fit(matrix)
91
92
        return pca
93
94
    # TODO: add the SVD method from section 6 step 1
95
    # It seems there is a mistake there, I think it is the same as PCA
96
    # just with repleacing it with SVD
97
    def _identify_direction(self, positive_end, negative_end,
98
                            definitional, method='pca'):
99
        if method not in DIRECTION_METHODS:
100
            raise ValueError('method should be one of {}, {} was given'.format(
101
                DIRECTION_METHODS, method))
102
103
        if positive_end == negative_end:
104
            raise ValueError('positive_end and negative_end'
105
                             'should be different, and not the same "{}"'
106
                             .format(positive_end))
107
108
        direction = None
109
110
        if method == 'single':
111
            direction = normalize(normalize(self[definitional[0]])
112
                                  - normalize(self[definitional[1]]))
113
114
        elif method == 'sum':
115
            groups = list(zip(*definitional))
116
117
            group1_sum_vector = np.sum([self[word]
118
                                        for word in groups[0]], axis=0)
119
            group2_sum_vector = np.sum([self[word]
120
                                        for word in groups[1]], axis=0)
121
122
            diff_vector = (normalize(group1_sum_vector)
123
                           - normalize(group2_sum_vector))
124
125
            direction = normalize(diff_vector)
126
127
        elif method == 'pca':
128
            pca = self._identify_subspace_by_pca(definitional, 1)
129
            if pca.explained_variance_ratio_[0] < FIRST_PC_THRESHOLD:
130
                raise RuntimeError('The Explained variance'
131
                                   'of the first principal component should be'
132
                                   'at least {}, but it is {}'
133
                                   .format(FIRST_PC_THRESHOLD,
134
                                           pca.explained_variance_ratio_[0]))
135
            direction = pca.components_[0]
136
137
        # if direction is oposite (e.g. we cannot control
138
        # what the PCA will return)
139
        ends_diff_projection = cosine_similarity((self[positive_end]
140
                                                  - self[negative_end]),
141
                                                 direction)
142
        if ends_diff_projection < 0:
143
            direction = -direction  # pylint: disable=invalid-unary-operand-type
144
145
        self.direction = direction
146
        self.positive_end = positive_end
147
        self.negative_end = negative_end
148
149
    def project_on_direction(self, word):
150
        self._is_direction_identified()
151
152
        vector = self[word]
153
        projection_score = self.model.cosine_similarities(self.direction,
154
                                                          [vector])[0]
155
        return projection_score
156
157
    def _calc_projection_scores(self, words):
158
        self._is_direction_identified()
159
160
        df = pd.DataFrame({'word': words})
161
162
        # TODO: maybe using cosine_similarities on all the vectors?
163
        # it might be faster
164
        df['projection'] = df['word'].apply(self.project_on_direction)
165
        df = df.sort_values('projection', ascending=False)
166
167
        return df
168
169
    def plot_projection_scores(self, words,
170
                               ax=None, axis_projection_step=None):
171
        self._is_direction_identified()
172
173
        projections_df = self._calc_projection_scores(words)
174
        projections_df['projection'] = projections_df['projection'].round(2)
175
176
        if ax is None:
177
            _, ax = plt.subplots(1)
178
179
        if axis_projection_step is None:
180
            axis_projection_step = 0.1
181
182
        cmap = plt.get_cmap('RdBu')
183
        projections_df['color'] = ((projections_df['projection'] + 0.5)
184
                                   .apply(cmap))
185
186
        most_extream_projection = (projections_df['projection']
187
                                   .abs()
188
                                   .max()
189
                                   .round(1))
190
191
        sns.barplot(x='projection', y='word', data=projections_df,
192
                    palette=projections_df['color'])
193
194
        plt.xticks(np.arange(-most_extream_projection, most_extream_projection,
195
                             axis_projection_step))
196
        plt.title('← {} {} {} →'.format(self.negative_end,
197
                                        ' ' * 20,
198
                                        self.positive_end))
199
200
        plt.xlabel('Direction Projection')
201
        plt.ylabel('Words')
202
203
    def calc_direct_bias(self, neutral_words, c=None):
204
        if c is None:
205
            c = 1
206
207
        projections = self._calc_projection_scores(neutral_words)['projection']
208
        direct_bias_terms = np.abs(projections) ** c
209
        direct_bias = direct_bias_terms.sum() / len(neutral_words)
210
211
        return direct_bias
212
213
    def calc_indirect_bias(self, word1, word2):
214
        """Also known in the article as PairBias."""
215
        self._is_direction_identified()
216
217
        vector1 = normalize(self[word1])
218
        vector2 = normalize(self[word2])
219
220
        perpendicular_vector1 = reject_vector(vector1, self.direction)
221
        perpendicular_vector2 = reject_vector(vector2, self.direction)
222
223
        inner_product = vector1 @ vector2
224
        perpendicular_similarity = cosine_similarity(perpendicular_vector1,
225
                                                     perpendicular_vector2)
226
227
        indirect_bias = ((inner_product - perpendicular_similarity)
228
                         / inner_product)
229
        return indirect_bias
230
231
    def _extract_neutral_words(self, specific_words):
232
        extended_specific_words = set()
233
234
        # because or specific_full data was trained on partial words embedding
235
        for word in specific_words:
236
            extended_specific_words.add(word)
237
            extended_specific_words.add(word.lower())
238
            extended_specific_words.add(word.upper())
239
            extended_specific_words.add(word.title())
240
241
        neutral_words = [word for word in self.model.vocab
242
                         if word not in extended_specific_words]
243
244
        return neutral_words
245
246
    def _neutralize(self, neutral_words, verbose=False):
247
        self._is_direction_identified()
248
249
        if verbose:
250
            neutral_words_iter = tqdm(neutral_words)
251
        else:
252
            neutral_words_iter = iter(neutral_words)
253
254
        for word in neutral_words_iter:
255
            neutralized_vector = reject_vector(self[word],
256
                                               self.direction)
257
            update_word_vector(self.model, word, neutralized_vector)
258
259
        self.model.init_sims(replace=True)
260
261
    def _equalize(self, equality_sets):
262
        for equality_set_words in equality_sets:
263
            equality_set_vectors = [normalize(self[word])
264
                                    for word in equality_set_words]
265
            center = np.mean(equality_set_vectors, axis=0)
266
            (projected_center,
267
             rejected_center) = project_reject_vector(center,
268
                                                      self.direction)
269
270
            for word, vector in zip(equality_set_words, equality_set_vectors):
271
                projected_vector = project_vector(vector, self.direction)
272
273
                projected_part = normalize(projected_vector - projected_center)
274
                scaling = np.sqrt(1 - np.linalg.norm(rejected_center)**2)
275
276
                # TODO - in the code it is different - why?
277
                # equalized_vector = rejected_center + scaling * self.direction
278
                # https://github.com/tolga-b/debiaswe/blob/10277b23e187ee4bd2b6872b507163ef4198686b/debiaswe/debias.py#L36-L37
279
                equalized_vector = rejected_center + scaling * projected_part
280
281
                update_word_vector(self.model, word, equalized_vector)
282
283
        self.model.init_sims(replace=True)
284
285
    def debias(self, method='hard', neutral_words=None, equality_sets=None,
286
               inplace=True, verbose=False):
287
        # pylint: disable=W0212
288
        if inplace:
289
            bias_words_embedding = self
290
        else:
291
            bias_words_embedding = copy.deepcopy(self)
292
293
        if method not in DEBIAS_METHODS:
294
            raise ValueError('method should be one of {}, {} was given'.format(
295
                DEBIAS_METHODS, method))
296
297
        if method in ['hard', 'neutralize']:
298
            if verbose:
299
                print('Neutralize...')
300
            bias_words_embedding._neutralize(neutral_words, verbose)
301
302
        if method == 'hard':
303
            if verbose:
304
                print('Equalize...')
305
            bias_words_embedding._equalize(equality_sets)
306
307
        if inplace:
308
            return None
309
        else:
310
            return bias_words_embedding
311
312
    def evaluate_words_embedding(self, verbose=False):
313
        with warnings.catch_warnings():
314
            warnings.simplefilter('ignore', category=FutureWarning)
315
316
            if verbose:
317
                print('Evaluate word pairs...')
318
            word_pairs_path = resource_filename(__name__,
319
                                                os.path.join('data',
320
                                                             'evaluation',
321
                                                             'wordsim353.tsv'))
322
            word_paris_result = self.model.evaluate_word_pairs(word_pairs_path)
323
324
            if verbose:
325
                print('Evaluate analogies...')
326
            analogies_path = resource_filename(__name__,
327
                                               os.path.join('data',
328
                                                            'evaluation',
329
                                                            'questions-words.txt'))  # pylint: disable=C0301
330
            analogies_result = self.model.evaluate_word_analogies(analogies_path)  # pylint: disable=C0301
331
332
        if verbose:
333
            print()
334
        print('From Gensim')
335
        print()
336
        print('-' * 30)
337
        print()
338
        print('Word Pairs Result - WordSimilarity-353:')
339
        print('~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
340
        print('Pearson correlation coefficient:', word_paris_result[0])
341
        print('Spearman rank-order correlation coefficient'
342
              'between the similarities from the dataset'
343
              'and the similarities produced by the model itself:',
344
              word_paris_result[1])
345
        print('Ratio of pairs with unknown words:', word_paris_result[2])
346
        print()
347
        print('-' * 30)
348
        print()
349
        print('Analogies Result')
350
        print('~~~~~~~~~~~~~~~~')
351
        print('Overall evaluation score:', analogies_result[0])
352
353
    def learn_full_specific_words(self, seed_specific_words,
354
                                  max_non_specific_examples=None, debug=None):
355
356
        if debug is None:
357
            debug = False
358
359
        if max_non_specific_examples is None:
360
            max_non_specific_examples = MAX_NON_SPECIFIC_EXAMPLES
361
362
        data = []
363
        non_specific_example_count = 0
364
365
        for word in self.model.vocab:
366
            is_specific = word in seed_specific_words
367
368
            if not is_specific:
369
                non_specific_example_count += 1
370
                if non_specific_example_count <= max_non_specific_examples:
371
                    data.append((self[word], is_specific))
372
            else:
373
                data.append((self[word], is_specific))
374
375
        np.random.seed(RANDOM_STATE)
376
        np.random.shuffle(data)
377
378
        X, y = zip(*data)
379
380
        X = np.array(X)
381
        X /= np.linalg.norm(X, axis=1)[:, None]
382
383
        y = np.array(y).astype('int')
384
385
        clf = LinearSVC(C=1, class_weight='balanced',
386
                        random_state=RANDOM_STATE)
387
388
        clf.fit(X, y)
389
390
        full_specific_words = []
391
        for word in self.model.vocab:
392
            vector = [normalize(self[word])]
393
            if clf.predict(vector):
394
                full_specific_words.append(word)
395
396
        if not debug:
397
            return full_specific_words, clf
398
399
        return full_specific_words, clf, X, y
400