excel.tests.test_helpers.ExcelTemplateReaderTestCase.test_column_not_found() - Code Metrics - Inspection of "Simplifying template loading process" - cnr-ibba/IMAGE-InjectTool - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Pull Request — master (#47)

by Paolo

created 2019-09-09 13:25 UTC

ExcelTemplateReaderTestCase.test_column_not_found() A

↳ Parent: excel.tests.test_helpers

Complexity

Conditions

Size

Total Lines	13
Code Lines	9

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
eloc	9
dl	0
loc	13
rs	9.95
c	0
b	0
f	0
cc	1
nop	1

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Jul  2 10:58:42 2019

@author: Paolo Cozzi <[email protected]>
"""

import types
from collections import defaultdict, namedtuple
from unittest.mock import patch, Mock

from django.test import TestCase

from common.tests import WebSocketMixin
from image_app.tests.mixins import (
    DataSourceMixinTestCase, FileReaderMixinTestCase)

from ..helpers import (
    ExcelTemplateReader, upload_template, TEMPLATE_COLUMNS, ExcelImportError)
from .common import BaseExcelMixin


class ExcelTemplateReaderTestCase(
        FileReaderMixinTestCase, DataSourceMixinTestCase, BaseExcelMixin,
        TestCase):
    """Test excel class upload"""

    def setUp(self):
        # calling my base class setup
        super().setUp()

        self.maxDiff = None

        # crate a Excel Template object
        self.reader = ExcelTemplateReader()

        # get filenames for DataSourceMixinTestCase.dst_path
        self.reader.read_file(self.dst_path)

    def test_check_sheets(self):
        # test check sheets method
        status, not_found = self.reader.check_sheets()

        self.assertTrue(status)
        self.assertEqual(not_found, [])

        # override sheet names
        self.reader.sheet_names = []

        # check method
        status, not_found = self.reader.check_sheets()

        self.assertFalse(status)
        self.assertEqual(not_found, ['breed', 'animal', 'sample'])

    def test_check_columns(self):
        # test check sheets method
        status, not_found = self.reader.check_columns()

        self.assertTrue(status)
        self.assertIsInstance(not_found, defaultdict)
        self.assertEqual(len(not_found), 0)

    @patch('xlrd.open_workbook')
    def test_check_columns_issue(self, mock_open):
        """Test a file with columns issues"""

        # creating a mock excel book
        mock_book = Mock()

        # customizing the mock object
        mock_book.sheet_names.return_value = ['breed', 'animal', 'sample']

        # creating a mock sheet for breed
        breed_sheet = Mock()
        breed_sheet.nrows = 1

        # now setting rows to a fake sheet
        breed_sheet.row_values.return_value = []

        # creating a mock sheet for animal
        animal_sheet = Mock()
        animal_sheet.nrows = 1

        # now setting rows to a fake sheet
        animal_sheet.row_values.return_value = []

        # creating a mock sheet for sample
        sample_sheet = Mock()
        sample_sheet.nrows = 1

        # now setting rows to a fake sheet
        sample_sheet.row_values.return_value = []

        # finally setting sheet to the fabe excel object
        mock_book.sheet_by_name.side_effect = [
            breed_sheet, animal_sheet, sample_sheet]

        # returning the mock object when opening a workbook
        mock_open.return_value = mock_book

        # now calling methods
        reader = ExcelTemplateReader()
        reader.read_file("fake file")

        # create the reference error output
        reference = defaultdict(list)
        reference['breed'] += TEMPLATE_COLUMNS['breed']
        reference['animal'] += TEMPLATE_COLUMNS['animal']
        reference['sample'] += TEMPLATE_COLUMNS['sample']

        status, test = reader.check_columns()

        self.assertFalse(status)
        self.assertIsInstance(test, defaultdict)
        self.assertDictEqual(reference, test)

    def check_generator(self, records, length):
        self.assertIsInstance(records, types.GeneratorType)
        self.assertEqual(len(list(records)), length)

    def test_get_breed_records(self):
        """get_breed_records returns an iterator"""

        breeds = self.reader.get_breed_records()
        self.check_generator(breeds, 2)

    def test_get_animal_records(self):
        """get_animal_records returns an iterator"""

        animals = self.reader.get_animal_records()
        self.check_generator(animals, 3)

    def test_get_sample_records(self):
        """get_sample_records returns an iterator"""

        samples = self.reader.get_sample_records()
        self.check_generator(samples, 3)

    def test_check_accuracies(self):
        """Test check accuracies method"""

        check, not_found = self.reader.check_accuracies()

        self.assertTrue(check)
        self.assertEqual(len(not_found), 0)

    @patch.dict("excel.helpers.exceltemplate.TEMPLATE_COLUMNS",
                {'breed': ["a column"]})
    def test_column_not_found(self):
        """Test a column not found raise an informativa exception"""

        # get a generator object
        generator = self.reader.get_sheet_records('breed')

        self.assertRaisesRegex(
            ExcelImportError,
            "Column 'a column' not found in 'breed' sheet",
            list,
            generator)

    @patch('xlrd.open_workbook')
    def test_check_accuracies_issue(self, mock_open):
        """Checking issues with accuracy in excels data"""

        # creating a mock excel book
        mock_book = Mock()

        # customizing the mock object
        mock_book.sheet_names.return_value = ['breed', 'animal', 'sample']

        # creating a mock sheet for animal
        animal_sheet = Mock()
        animal_sheet.nrows = 2

        # creating a fake row of data
        fake_row = ["" for col in TEMPLATE_COLUMNS['animal']]

        # get birth location accuracy index
        accuracy_idx = TEMPLATE_COLUMNS['animal'].index(
            "Birth location accuracy")

        # set a fake accuracy item
        fake_row[accuracy_idx] = "Fake"

        # now setting rows to a fake sheet
        animal_sheet.row_values.side_effect = [
            TEMPLATE_COLUMNS['animal'],
            fake_row]

        # creating a mock sheet for sample
        sample_sheet = Mock()
        sample_sheet.nrows = 2

        # creating a fake row of data
        fake_row = ["" for col in TEMPLATE_COLUMNS['sample']]

        # get birth location accuracy index
        accuracy_idx = TEMPLATE_COLUMNS['sample'].index(
            "Collection place accuracy")

        # set a fake accuracy item
        fake_row[accuracy_idx] = "Fake"

        # now setting rows to a fake sheet
        sample_sheet.row_values.side_effect = [
            TEMPLATE_COLUMNS['sample'],
            fake_row]

        # finally setting sheet to the fabe excel object
        mock_book.sheet_by_name.side_effect = [animal_sheet, sample_sheet]

        # returning the mock object when opening a workbook
        mock_open.return_value = mock_book

        # now calling methods
        reader = ExcelTemplateReader()
        reader.read_file("fake file")

        # define the expected value
        reference = (False, set(["Fake"]))
        test = reader.check_accuracies()

        self.assertEqual(reference, test)

    @patch('excel.helpers.ExcelTemplateReader.get_animal_records')
    def test_species_in_animal_and_breeds_differ(self, my_animal):
        # creating a fake row of data
        fake_row = ["" for col in TEMPLATE_COLUMNS['animal']]

        # get birth location accuracy index
        specie_idx = TEMPLATE_COLUMNS['animal'].index(
            "Species")

        # set a fake specie item
        fake_row[specie_idx] = "Fake"

        # create a namedtuple object
        columns = [col.lower().replace(" ", "_")
                   for col in TEMPLATE_COLUMNS['animal']]
        Record = namedtuple("Animal", columns)
        record = Record._make(fake_row)
        my_animal.return_value = [record]

        # define the expected value
        reference = (False, ["Fake"])
        test = self.reader.check_species_in_animal_sheet()

        self.assertEqual(reference, test)


class ExcelMixin(DataSourceMixinTestCase, WebSocketMixin, BaseExcelMixin):
    """Common tests for Excel classes"""

    # define the method to upload data from. Since the function is now inside
    # a class it becomes a method, specifically a bound method and is supposed
    # to receive the self attribute by default. If we don't want to get the
    # self attribute, we have to declare function as a staticmetho
    # https://stackoverflow.com/a/35322635/4385116
    upload_method = staticmethod(upload_template)

    def test_upload_template(self):
        """Testing uploading and importing data from excel template to UID"""

        # test data loaded
        message = "Template import completed for submission"
        self.upload_datasource(message)

        # check async message called
        notification_message = (
            'Template import completed for submission: 1')
        validation_message = {
            'animals': 3, 'samples': 3,
            'animal_unkn': 3, 'sample_unkn': 3,
            'animal_issues': 0, 'sample_issues': 0}

        # check async message called using WebSocketMixin.check_message
        self.check_message('Loaded', notification_message, validation_message)

    def check_errors(self, my_check, message, notification_message):
        """Common stuff for error in excel template loading"""

        super().check_errors(my_check, message)

        # check async message called using WebSocketMixin.check_message
        self.check_message('Error', notification_message)


class UploadTemplateTestCase(ExcelMixin, TestCase):
    """Test uploading data for Template excel path"""

    @patch("excel.helpers.ExcelTemplateReader.check_species",
           return_value=[False, 'Rainbow trout'])
    def test_upload_template_errors_with_species(self, my_check):
        """Testing importing with data into UID with errors in species"""

        message = "Some species are not loaded in UID database"
        notification_message = (
            'Error in importing data: Some species '
            'are not loaded in UID database: Rainbow '
            'trout')

        # check template import fails
        self.check_errors(my_check, message, notification_message)

    @patch("excel.helpers.ExcelTemplateReader.check_species_in_animal_sheet",
           return_value=[False, 'Rainbow trout'])
    def test_upload_template_errors_with_species_in_animal(self, my_check):
        """Testing importing with data into UID with errors animal"""

        message = "Some species are not defined in breed sheet"
        notification_message = (
            'Error in importing data: %s: Rainbow '
            'trout' % message)

        # check template import fails
        self.check_errors(my_check, message, notification_message)

    @patch("excel.helpers.ExcelTemplateReader.check_sex",
           return_value=[False, 'unknown'])
    def test_upload_template_errors_with_sex(self, my_check):
        """Testing importing with data into UID with errors"""

        message = "Not all Sex terms are loaded into database"
        notification_message = (
            'Error in importing data: Not all Sex '
            'terms are loaded into database: check '
            'for unknown in your dataset')

        # check template import fails
        self.check_errors(my_check, message, notification_message)

    @patch("excel.helpers.ExcelTemplateReader.check_accuracies",
           return_value=(False, set(["Fake"])))
    def test_upload_template_errors_with_accuracies(self, my_check):
        """Testing importing with data into UID with errors"""

        message = "Not all accuracy levels are defined in database"
        notification_message = (
            "Error in importing data: Not all accuracy "
            "levels are defined in database: check "
            "for {'Fake'} in your dataset")

        # check template import fails
        self.check_errors(my_check, message, notification_message)


class ReloadTemplateTestCase(ExcelMixin, TestCase):
    """Simulate a template reload case. Load data as in
    UploadTemplateTestCase, then call test which reload the same data"""

    # override used fixtures
    fixtures = [
        'crbanim/auth',
        'excel/dictspecie',
        'excel/image_app',
        'excel/submission',
        'excel/speciesynonym'
    ]


1			#!/usr/bin/env python3
2			# -- coding: utf-8 --
3			"""
4			Created on Tue Jul 2 10:58:42 2019
5
6			@author: Paolo Cozzi <[email protected]>
7			"""
8
9			import types
10			from collections import defaultdict, namedtuple
11			from unittest.mock import patch, Mock
12
13			from django.test import TestCase
14
15			from common.tests import WebSocketMixin
16			from image_app.tests.mixins import (
17			DataSourceMixinTestCase, FileReaderMixinTestCase)
18
19			from ..helpers import (
20			ExcelTemplateReader, upload_template, TEMPLATE_COLUMNS, ExcelImportError)
21			from .common import BaseExcelMixin
22
23
24			class ExcelTemplateReaderTestCase(
25			FileReaderMixinTestCase, DataSourceMixinTestCase, BaseExcelMixin,
26			TestCase):
27			"""Test excel class upload"""
28
29			def setUp(self):
30			# calling my base class setup
31			super().setUp()
32
33			self.maxDiff = None
34
35			# crate a Excel Template object
36			self.reader = ExcelTemplateReader()
37
38			# get filenames for DataSourceMixinTestCase.dst_path
39			self.reader.read_file(self.dst_path)
40
41			def test_check_sheets(self):
42			# test check sheets method
43			status, not_found = self.reader.check_sheets()
44
45			self.assertTrue(status)
46			self.assertEqual(not_found, [])
47
48			# override sheet names
49			self.reader.sheet_names = []
50
51			# check method
52			status, not_found = self.reader.check_sheets()
53
54			self.assertFalse(status)
55			self.assertEqual(not_found, ['breed', 'animal', 'sample'])
56
57			def test_check_columns(self):
58			# test check sheets method
59			status, not_found = self.reader.check_columns()
60
61			self.assertTrue(status)
62			self.assertIsInstance(not_found, defaultdict)
63			self.assertEqual(len(not_found), 0)
64
65			@patch('xlrd.open_workbook')
66			def test_check_columns_issue(self, mock_open):
67			"""Test a file with columns issues"""
68
69			# creating a mock excel book
70			mock_book = Mock()
71
72			# customizing the mock object
73			mock_book.sheet_names.return_value = ['breed', 'animal', 'sample']
74
75			# creating a mock sheet for breed
76			breed_sheet = Mock()
77			breed_sheet.nrows = 1
78
79			# now setting rows to a fake sheet
80			breed_sheet.row_values.return_value = []
81
82			# creating a mock sheet for animal
83			animal_sheet = Mock()
84			animal_sheet.nrows = 1
85
86			# now setting rows to a fake sheet
87			animal_sheet.row_values.return_value = []
88
89			# creating a mock sheet for sample
90			sample_sheet = Mock()
91			sample_sheet.nrows = 1
92
93			# now setting rows to a fake sheet
94			sample_sheet.row_values.return_value = []
95
96			# finally setting sheet to the fabe excel object
97			mock_book.sheet_by_name.side_effect = [
98			breed_sheet, animal_sheet, sample_sheet]
99
100			# returning the mock object when opening a workbook
101			mock_open.return_value = mock_book
102
103			# now calling methods
104			reader = ExcelTemplateReader()
105			reader.read_file("fake file")
106
107			# create the reference error output
108			reference = defaultdict(list)
109			reference['breed'] += TEMPLATE_COLUMNS['breed']
110			reference['animal'] += TEMPLATE_COLUMNS['animal']
111			reference['sample'] += TEMPLATE_COLUMNS['sample']
112
113			status, test = reader.check_columns()
114
115			self.assertFalse(status)
116			self.assertIsInstance(test, defaultdict)
117			self.assertDictEqual(reference, test)
118
119			def check_generator(self, records, length):
120			self.assertIsInstance(records, types.GeneratorType)
121			self.assertEqual(len(list(records)), length)
122
123			def test_get_breed_records(self):
124			"""get_breed_records returns an iterator"""
125
126			breeds = self.reader.get_breed_records()
127			self.check_generator(breeds, 2)
128
129			def test_get_animal_records(self):
130			"""get_animal_records returns an iterator"""
131
132			animals = self.reader.get_animal_records()
133			self.check_generator(animals, 3)
134
135			def test_get_sample_records(self):
136			"""get_sample_records returns an iterator"""
137
138			samples = self.reader.get_sample_records()
139			self.check_generator(samples, 3)
140
141			def test_check_accuracies(self):
142			"""Test check accuracies method"""
143
144			check, not_found = self.reader.check_accuracies()
145
146			self.assertTrue(check)
147			self.assertEqual(len(not_found), 0)
148
149			@patch.dict("excel.helpers.exceltemplate.TEMPLATE_COLUMNS",
150			{'breed': ["a column"]})
151			def test_column_not_found(self):
152			"""Test a column not found raise an informativa exception"""
153
154			# get a generator object
155			generator = self.reader.get_sheet_records('breed')
156
157			self.assertRaisesRegex(
158			ExcelImportError,
159			"Column 'a column' not found in 'breed' sheet",
160			list,
161			generator)
162
163			@patch('xlrd.open_workbook')
164			def test_check_accuracies_issue(self, mock_open):
165			"""Checking issues with accuracy in excels data"""
166
167			# creating a mock excel book
168			mock_book = Mock()
169
170			# customizing the mock object
171			mock_book.sheet_names.return_value = ['breed', 'animal', 'sample']
172
173			# creating a mock sheet for animal
174			animal_sheet = Mock()
175			animal_sheet.nrows = 2
176
177			# creating a fake row of data
178			fake_row = ["" for col in TEMPLATE_COLUMNS['animal']]
179
180			# get birth location accuracy index
181			accuracy_idx = TEMPLATE_COLUMNS['animal'].index(
182			"Birth location accuracy")
183
184			# set a fake accuracy item
185			fake_row[accuracy_idx] = "Fake"
186
187			# now setting rows to a fake sheet
188			animal_sheet.row_values.side_effect = [
189			TEMPLATE_COLUMNS['animal'],
190			fake_row]
191
192			# creating a mock sheet for sample
193			sample_sheet = Mock()
194			sample_sheet.nrows = 2
195
196			# creating a fake row of data
197			fake_row = ["" for col in TEMPLATE_COLUMNS['sample']]
198
199			# get birth location accuracy index
200			accuracy_idx = TEMPLATE_COLUMNS['sample'].index(
201			"Collection place accuracy")
202
203			# set a fake accuracy item
204			fake_row[accuracy_idx] = "Fake"
205
206			# now setting rows to a fake sheet
207			sample_sheet.row_values.side_effect = [
208			TEMPLATE_COLUMNS['sample'],
209			fake_row]
210
211			# finally setting sheet to the fabe excel object
212			mock_book.sheet_by_name.side_effect = [animal_sheet, sample_sheet]
213
214			# returning the mock object when opening a workbook
215			mock_open.return_value = mock_book
216
217			# now calling methods
218			reader = ExcelTemplateReader()
219			reader.read_file("fake file")
220
221			# define the expected value
222			reference = (False, set(["Fake"]))
223			test = reader.check_accuracies()
224
225			self.assertEqual(reference, test)
226
227			@patch('excel.helpers.ExcelTemplateReader.get_animal_records')
228			def test_species_in_animal_and_breeds_differ(self, my_animal):
229			# creating a fake row of data
230			fake_row = ["" for col in TEMPLATE_COLUMNS['animal']]
231
232			# get birth location accuracy index
233			specie_idx = TEMPLATE_COLUMNS['animal'].index(
234			"Species")
235
236			# set a fake specie item
237			fake_row[specie_idx] = "Fake"
238
239			# create a namedtuple object
240			columns = [col.lower().replace(" ", "_")
241			for col in TEMPLATE_COLUMNS['animal']]
242			Record = namedtuple("Animal", columns)
243			record = Record._make(fake_row)
244			my_animal.return_value = [record]
245
246			# define the expected value
247			reference = (False, ["Fake"])
248			test = self.reader.check_species_in_animal_sheet()
249
250			self.assertEqual(reference, test)
251
252
253			class ExcelMixin(DataSourceMixinTestCase, WebSocketMixin, BaseExcelMixin):
254			"""Common tests for Excel classes"""
255
256			# define the method to upload data from. Since the function is now inside
257			# a class it becomes a method, specifically a bound method and is supposed
258			# to receive the self attribute by default. If we don't want to get the
259			# self attribute, we have to declare function as a staticmetho
260			# https://stackoverflow.com/a/35322635/4385116
261			upload_method = staticmethod(upload_template)
262
263			def test_upload_template(self):
264			"""Testing uploading and importing data from excel template to UID"""
265
266			# test data loaded
267			message = "Template import completed for submission"
268			self.upload_datasource(message)
269
270			# check async message called
271			notification_message = (
272			'Template import completed for submission: 1')
273			validation_message = {
274			'animals': 3, 'samples': 3,
275			'animal_unkn': 3, 'sample_unkn': 3,
276			'animal_issues': 0, 'sample_issues': 0}
277
278			# check async message called using WebSocketMixin.check_message
279			self.check_message('Loaded', notification_message, validation_message)
280
281			def check_errors(self, my_check, message, notification_message):
282			"""Common stuff for error in excel template loading"""
283
284			super().check_errors(my_check, message)
285
286			# check async message called using WebSocketMixin.check_message
287			self.check_message('Error', notification_message)
288
289
290			class UploadTemplateTestCase(ExcelMixin, TestCase):
291			"""Test uploading data for Template excel path"""
292
293			@patch("excel.helpers.ExcelTemplateReader.check_species",
294			return_value=[False, 'Rainbow trout'])
295			def test_upload_template_errors_with_species(self, my_check):
296			"""Testing importing with data into UID with errors in species"""
297
298			message = "Some species are not loaded in UID database"
299			notification_message = (
300			'Error in importing data: Some species '
301			'are not loaded in UID database: Rainbow '
302			'trout')
303
304			# check template import fails
305			self.check_errors(my_check, message, notification_message)
306
307			@patch("excel.helpers.ExcelTemplateReader.check_species_in_animal_sheet",
308			return_value=[False, 'Rainbow trout'])
309			def test_upload_template_errors_with_species_in_animal(self, my_check):
310			"""Testing importing with data into UID with errors animal"""
311
312			message = "Some species are not defined in breed sheet"
313			notification_message = (
314			'Error in importing data: %s: Rainbow '
315			'trout' % message)
316
317			# check template import fails
318			self.check_errors(my_check, message, notification_message)
319
320			@patch("excel.helpers.ExcelTemplateReader.check_sex",
321			return_value=[False, 'unknown'])
322			def test_upload_template_errors_with_sex(self, my_check):
323			"""Testing importing with data into UID with errors"""
324
325			message = "Not all Sex terms are loaded into database"
326			notification_message = (
327			'Error in importing data: Not all Sex '
328			'terms are loaded into database: check '
329			'for unknown in your dataset')
330
331			# check template import fails
332			self.check_errors(my_check, message, notification_message)
333
334			@patch("excel.helpers.ExcelTemplateReader.check_accuracies",
335			return_value=(False, set(["Fake"])))
336			def test_upload_template_errors_with_accuracies(self, my_check):
337			"""Testing importing with data into UID with errors"""
338
339			message = "Not all accuracy levels are defined in database"
340			notification_message = (
341			"Error in importing data: Not all accuracy "
342			"levels are defined in database: check "
343			"for {'Fake'} in your dataset")
344
345			# check template import fails
346			self.check_errors(my_check, message, notification_message)
347
348
349			class ReloadTemplateTestCase(ExcelMixin, TestCase):
350			"""Simulate a template reload case. Load data as in
351			UploadTemplateTestCase, then call test which reload the same data"""
352
353			# override used fixtures
354			fixtures = [
355			'crbanim/auth',
356			'excel/dictspecie',
357			'excel/image_app',
358			'excel/submission',
359			'excel/speciesynonym'
360			]
361

cnr-ibba / IMAGE-InjectTool

Pull Request — master (#47)

ExcelTemplateReaderTestCase.test_column_not_found() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like