ProjectTest - Code Metrics - Inspection of "fixed error in LDA (transposed score/topic)" - ContinuumIO/topik - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Pull Request — master (#77)

unknown

created 2016-04-19 13:40 UTC

ProjectTest A

↳ Parent: Project

Complexity

Total Complexity

Size/Duplication

Total Lines	67
Duplicated Lines	0 %

Metric	Value
wmc	18
dl	0
loc	67
rs	10

8 Methods

Rating	Name	Size	Complexity
A	test_read_input()	2	1
A	test_model()	8	3
A	test_vectorize()	5	1
A	test_get_filtered_corpus_iterator()	4	1
A	test_get_date_filtered_corpus_iterator()	4	1
C	test_context_manager()	23	7
A	test_visualize()	5	1
A	test_tokenize()	8	3

import glob
class SomeClass:
    def some_method(self):
        """Do x and return foo."""
import os
import time
import unittest

import elasticsearch
import nose.tools as nt
# .scrutinizer.yml
before_commands:
    - sudo pip install abc # Python2
    - sudo pip3 install abc # Python3

from topik.fileio import TopikProject
from topik.fileio.tests import test_data_path

# make logging quiet during testing, to keep Travis CI logs short.
import logging
logging.basicConfig()
logging.getLogger('elasticsearch').setLevel(logging.ERROR)
logging.getLogger('urllib3').setLevel(logging.ERROR)

SAVE_FILENAME = "test_project"

sample_tokenized_doc = (2318580746137828354,

 [u'nano', u'sized', u'tio', u'particles', u'applications', u'including',
  u'use', u'photocatalysts', u'heat', u'transfer', u'fluids', u'nanofluids',
  u'present', u'study', u'tio', u'nanoparticles', u'controllable', u'phase',
  u'particle', u'size', u'obtained', u'homogeneous', u'gas', u'phase',
  u'nucleation', u'chemical', u'vapor', u'condensation', u'cvc', u'phase',
  u'particle', u'size', u'tio', u'nanoparticles', u'processing', u'conditions',
  u'characterized', u'x', u'ray', u'diffraction', u'transmission', u'electron',
  u'microscopy', u'chamber', u'temperature', u'pressure', u'key', u'parameters',
  u'affecting', u'particle', u'phase', u'size', u'pure', u'anatase', u'phase',
  u'observed', u'synthesis', u'temperatures', u'low', u'c', u'chamber',
  u'pressure', u'varying', u'torr', u'furnace', u'temperature', u'increased',
  u'c', u'pressure', u'torr', u'mixture', u'anatase', u'rutile', u'phases',
  u'observed', u'predominant', u'phase', u'anatase', u'average', u'particle',
  u'size', u'experimental', u'conditions', u'observed', u'nm'])

test_data_path = os.path.join(test_data_path, "test_data_json_stream.json")


class ProjectTest(object):
class SomeClass:
    def some_method(self):
        """Do x and return foo."""
    def test_context_manager(self):
class SomeClass:
    def some_method(self):
        """Do x and return foo."""
        for filename in glob.glob("context_output*"):
            os.remove(filename)
        with TopikProject("context_output", self.output_type, self.output_args) as project:

            project.read_input(source=test_data_path, content_field='abstract')
            project.tokenize()
            project.vectorize(method='bag_of_words')
            project.run_model(model_name='lda', ntopics=2)

        # above runs through a whole workflow (minus plotting.)  At end, it closes file.
        # load output here.
        with TopikProject("context_output") as project:
            nt.assert_equal(len(list(project.get_filtered_corpus_iterator())), 100)
            nt.assert_true(sample_tokenized_doc in list(iter(project.selected_tokenized_corpus)))
            nt.assert_equal(project.selected_vectorized_corpus.global_term_count, 2434)
            nt.assert_equal(len(project.selected_vectorized_corpus), 100)  # All documents processed
            for doc in project.selected_modeled_corpus.doc_topic_matrix.values():
                nt.assert_almost_equal(sum(doc), 1)
            for topic in project.selected_modeled_corpus.topic_term_matrix.values():
                nt.assert_almost_equal(sum(topic), 1)

        for filename in glob.glob("context_output*"):
            os.remove(filename)

    def test_read_input(self):
class SomeClass:
    def some_method(self):
        """Do x and return foo."""
        nt.assert_equal(len(list(self.project.get_filtered_corpus_iterator())), 100)


    def test_get_filtered_corpus_iterator(self):
class SomeClass:
    def some_method(self):
        """Do x and return foo."""
        doc_list = list(self.project.get_filtered_corpus_iterator())

        nt.assert_equal(type(doc_list[0]), type(('123', 'text')))
        nt.assert_equal(len(doc_list), 100)

    def test_get_date_filtered_corpus_iterator(self):
class SomeClass:
    def some_method(self):
        """Do x and return foo."""
        results = list(self.project.get_date_filtered_corpus_iterator(

            field_to_get="abstract", start=1975, end=1999, filter_field='year'))
        nt.assert_equal(len(results), 25)

    def test_tokenize(self):
class SomeClass:
    def some_method(self):
        """Do x and return foo."""
        self.project.tokenize('simple')

        in_results = False
        for id, doc in self.project.selected_tokenized_corpus:

            if doc in sample_tokenized_doc:
                in_results = True
                break
        nt.assert_true(in_results)

    def test_vectorize(self):
class SomeClass:
    def some_method(self):
        """Do x and return foo."""
        self.project.tokenize()

        self.project.vectorize()

        nt.assert_equal(self.project.selected_vectorized_corpus.global_term_count, 2434)

        nt.assert_equal(len(self.project.selected_vectorized_corpus), 100)  # All documents processed


    def test_model(self):
class SomeClass:
    def some_method(self):
        """Do x and return foo."""
        self.project.tokenize()

        self.project.vectorize()

        self.project.run_model(model_name='lda', ntopics=2)

        for doc in self.project.selected_modeled_corpus.doc_topic_matrix.values():

            nt.assert_almost_equal(sum(doc), 1)
        for topic in self.project.selected_modeled_corpus.topic_term_matrix.values():

            nt.assert_almost_equal(sum(topic), 1)

    def test_visualize(self):
class SomeClass:
    def some_method(self):
        """Do x and return foo."""
        self.project.tokenize()

        self.project.vectorize(method='bag_of_words')

        self.project.run_model(ntopics=2)

        self.project.visualize(vis_name='termite', topn=5)



class TestInMemoryOutput(unittest.TestCase, ProjectTest):
class SomeClass:
    def some_method(self):
        """Do x and return foo."""
    def setUp(self):
        self.output_type = "InMemoryOutput"
        self.output_args = {}
        self.project = TopikProject("test_project",
                                    output_type=self.output_type,
                                    output_args=self.output_args)
        self.project.read_input(test_data_path, content_field="abstract")

class TestElasticSearchOutput(unittest.TestCase, ProjectTest):
class SomeClass:
    def some_method(self):
        """Do x and return foo."""
    INDEX = "test_index"
    def setUp(self):
        self.output_type = "ElasticSearchOutput"
        self.output_args = {'source': 'localhost',
                            'index': TestElasticSearchOutput.INDEX,
                            'content_field': "abstract"}
        self.project = TopikProject("test_project", output_type=self.output_type,
                                    output_args=self.output_args)
        self.project.read_input(test_data_path, content_field="abstract",
                                synchronous_wait=30)

    def tearDown(self):
        instance = elasticsearch.Elasticsearch("localhost")
        instance.indices.delete(TestElasticSearchOutput.INDEX)
        if instance.indices.exists("{}_year_alias_date".format(TestElasticSearchOutput.INDEX)):
            instance.indices.delete("{}_year_alias_date".format(TestElasticSearchOutput.INDEX))
        time.sleep(1)


Pull Request — master (#77)

ProjectTest A

Complexity

Size/Duplication

8 Methods

1. Missing Dependencies

2. Missing init.py files

1			import glob
			0 ignored issues – show Coding Style introduced 2015-11-23 14:51 UTC by Report Bug Copy Issue Report This module should have a docstring. The coding style of this project requires that you add a docstring to this code element. Below, you find an example for methods: class SomeClass: def some_method(self): """Do x and return foo.""" If you would like to know more about docstrings, we recommend to read PEP-257: Docstring Conventions. Loading history...
2			import os
3			import time
4			import unittest
5
6			import elasticsearch
7			import nose.tools as nt
			0 ignored issues – show Configuration introduced 2015-11-23 14:51 UTC by Report Bug Copy Issue Report The import `nose.tools` could not be resolved. This can be caused by one of the following: 1. Missing Dependencies This error could indicate a configuration issue of Pylint. Make sure that your libraries are available by adding the necessary commands. # .scrutinizer.yml before_commands: - sudo pip install abc # Python2 - sudo pip3 install abc # Python3 Tip: We are currently not using virtualenv to run pylint, when installing your modules make sure to use the command for the correct version. 2. Missing __init__.py files This error could also result from missing `__init__.py` files in your module folders. Make sure that you place one file in each sub-folder. Loading history...
8
9			from topik.fileio import TopikProject
10			from topik.fileio.tests import test_data_path
11
12			# make logging quiet during testing, to keep Travis CI logs short.
13			import logging
14			logging.basicConfig()
15			logging.getLogger('elasticsearch').setLevel(logging.ERROR)
16			logging.getLogger('urllib3').setLevel(logging.ERROR)
17
18			SAVE_FILENAME = "test_project"
19
20			sample_tokenized_doc = (2318580746137828354,
			0 ignored issues – show Coding Style Naming introduced 2015-11-23 14:51 UTC by Report Bug Copy Issue Report The name `sample_tokenized_doc` does not conform to the constant naming conventions (`(([A-Z_][A-Z0-9_])\|(__.__))$`). This check looks for invalid names for a range of different identifiers. You can set regular expressions to which the identifiers must conform if the defaults do not match your requirements. If your project includes a Pylint configuration file, the settings contained in that file take precedence. To find out more about Pylint, please refer to their site. Loading history...
21			[u'nano', u'sized', u'tio', u'particles', u'applications', u'including',
22			u'use', u'photocatalysts', u'heat', u'transfer', u'fluids', u'nanofluids',
23			u'present', u'study', u'tio', u'nanoparticles', u'controllable', u'phase',
24			u'particle', u'size', u'obtained', u'homogeneous', u'gas', u'phase',
25			u'nucleation', u'chemical', u'vapor', u'condensation', u'cvc', u'phase',
26			u'particle', u'size', u'tio', u'nanoparticles', u'processing', u'conditions',
27			u'characterized', u'x', u'ray', u'diffraction', u'transmission', u'electron',
28			u'microscopy', u'chamber', u'temperature', u'pressure', u'key', u'parameters',
29			u'affecting', u'particle', u'phase', u'size', u'pure', u'anatase', u'phase',
30			u'observed', u'synthesis', u'temperatures', u'low', u'c', u'chamber',
31			u'pressure', u'varying', u'torr', u'furnace', u'temperature', u'increased',
32			u'c', u'pressure', u'torr', u'mixture', u'anatase', u'rutile', u'phases',
33			u'observed', u'predominant', u'phase', u'anatase', u'average', u'particle',
34			u'size', u'experimental', u'conditions', u'observed', u'nm'])
35
36			test_data_path = os.path.join(test_data_path, "test_data_json_stream.json")
			0 ignored issues – show Coding Style Naming introduced 2015-11-23 14:51 UTC by Report Bug Copy Issue Report The name `test_data_path` does not conform to the constant naming conventions (`(([A-Z_][A-Z0-9_])\|(__.__))$`). This check looks for invalid names for a range of different identifiers. You can set regular expressions to which the identifiers must conform if the defaults do not match your requirements. If your project includes a Pylint configuration file, the settings contained in that file take precedence. To find out more about Pylint, please refer to their site. Loading history...

ContinuumIO / topik

Pull Request — master (#77)

ProjectTest A

Complexity

Size/Duplication

8 Methods

1. Missing Dependencies

2. Missing __init__.py files

Duplication Side-by-Side

Filter issues like

2. Missing init.py files