tracking_policy_agendas.word2vec.w2v_emb

Complexity

Total Complexity

13

Size/Duplication

Total Lines	53
Duplicated Lines	79.25 %

Importance

Changes

0

8 Methods

Rating	Name	Duplication	Size	Complexity
A	W2VEmb.__init()	5	5	1
A	W2VEmb.__init__()	5	5	2
A	W2VEmb.tf_idf_transformer()	6	6	1
A	W2VEmb.__getitem__()	3	3	2
A	W2VEmb.tf_idf_mean()	4	4	2
A	W2VEmb.load()	3	3	2
A	W2VEmb.save()	3	3	2
A	W2VEmb.encode()	5	5	1

import pickle

Missing module docstring

import gensim
import numpy as np
import pandas as pd
from gensim import utils
from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from .w2v_corpus import W2VCorpus


class W2VEmb:

Missing class docstring

This code seems to be duplicated in your project.

    def __init__(self, text_document=None):
        self.wv2_corpus = None
        self.w2v_model = None
        self.tf_idf_transformation = None
        if text_document is not None: self.__init(text_document)

More than one statement on a single line

    def __init(self, text_document: pd.Series):
        text_document = text_document.fillna('')
        self.tf_idf_transformation = self.tf_idf_transformer(text_document)
        self.wv2_corpus = W2VCorpus(text_document)
        self.w2v_model = gensim.models.Word2Vec(sentences=self.wv2_corpus, min_count=1, vector_size=900, epochs=50)

This line is too long as per the coding-style (115/100).

    def __getitem__(self, text: str) -> np.ndarray:
        try:    return self.w2v_model.wv[text]

More than one statement on a single line

Exactly one space required after :

        except: return np.array([0 for _ in range(0, self.w2v_model.vector_size)])

More than one statement on a single line

General except handlers without types should be used sparingly.

    def tf_idf_transformer(self, text_series):

This method could be written as a function/class method.

Missing function or method docstring

        tfidf = Pipeline([('count', CountVectorizer(encoding='utf-8', min_df=3, #max_df=0.9,
                                                    max_features=900,
                                                    ngram_range=(1, 2))),
                          ('tfid', TfidfTransformer(sublinear_tf=True, norm='l2'))]).fit(text_series.ravel())

This line is too long as per the coding-style (109/100).

        return tfidf

    def encode(self, text: str) -> np.ndarray:

Missing function or method docstring

        stream = utils.simple_preprocess(text)
        tf_idf_vec = self.tf_idf_transformation.transform(stream).toarray()
        w2v_encode = self[stream]
        return np.mean(list(self.tf_idf_mean(tf_idf_vec, w2v_encode)), axis=0)

    def save(self, path: str):

Missing function or method docstring

        with open(path, 'wb') as f:

Variable name "f" doesn't conform to snake_case naming style ('([^\\W\\dA-Z][^\\WA-Z]2,|_[^\\WA-Z]*|__[^\\WA-Z\\d_][^\\WA-Z]+__)$' pattern)

            pickle.dump(self, f, protocol=pickle.HIGHEST_PROTOCOL)

    def load(self, path: str):

Missing function or method docstring

        with open(path, 'rb') as f:

Variable name "f" doesn't conform to snake_case naming style ('([^\\W\\dA-Z][^\\WA-Z]2,|_[^\\WA-Z]*|__[^\\WA-Z\\d_][^\\WA-Z]+__)$' pattern)

            self.__dict__.update(pickle.load(f).__dict__)

    @staticmethod
    def tf_idf_mean(tf_idf_vec: np.ndarray, w2v_encode: np.ndarray):

Missing function or method docstring

        for ind in range(len(tf_idf_vec)):

Consider using enumerate instead of iterating with range and len

            yield tf_idf_vec[ind]*w2v_encode[ind]