biosample.tests.test_async_biosample.AsyncBioSamplesTestCase.test_server_lost() - Code Metrics - Inspection of ":twisted_rightwards_arrows: Merge pull request #97..." - cnr-ibba/IMAGE-InjectTool - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( 55dedd...16663f )

by Paolo

created 2020-06-08 15:56 UTC

AsyncBioSamplesTestCase.test_server_lost() A

↳ Parent: biosample.tests.test_async_biosample

Complexity

Conditions

Size

Total Lines	15
Code Lines	9

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
eloc	9
dl	0
loc	15
rs	9.95
c	0
b	0
f	0
cc	3
nop	1

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Jan 21 10:54:09 2020

@author: Paolo Cozzi <[email protected]>
"""

import os
import json
import time
import types
import asynctest

from aioresponses import aioresponses
from aiohttp.client_exceptions import ServerDisconnectedError
from unittest.mock import patch, Mock

from django.test import TestCase
from django.utils import timezone

from common.constants import BIOSAMPLE_URL, SUBMITTED, READY, COMPLETED
from uid.models import Animal as UIDAnimal, Sample as UIDSample

from ..tasks.cleanup import check_samples, get_orphan_samples, PAGE_SIZE
from ..models import OrphanSample, ManagedTeam

from .common import generate_token, BioSamplesMixin

# get my path
dir_path = os.path.dirname(os.path.realpath(__file__))

# define data path
DATA_PATH = os.path.join(dir_path, "data")


with open(os.path.join(DATA_PATH, "page_0.json")) as handle:
    page0 = handle.read()

with open(os.path.join(DATA_PATH, "page_1.json")) as handle:
    page1 = handle.read()

with open(os.path.join(DATA_PATH, "issue_page1.json")) as handle:
    issue_page1 = handle.read()


class AsyncBioSamplesTestCase(asynctest.TestCase, TestCase):

    fixtures = [
        'biosample/managedteam',
        'uid/animal',
        'uid/dictbreed',
        'uid/dictcountry',
        'uid/dictrole',
        'uid/dictsex',
        'uid/dictspecie',
        'uid/dictstage',
        'uid/dictuberon',
        'uid/ontology',
        'uid/organization',
        'uid/publication',
        'uid/sample',
        'uid/submission',
        'uid/user'
    ]

    @classmethod
    def setUpClass(cls):
        # calling my base class setup
        super().setUpClass()

        cls.mock_auth_patcher = patch('pyUSIrest.auth.requests.get')
        cls.mock_auth = cls.mock_auth_patcher.start()

    @classmethod
    def tearDownClass(cls):
        cls.mock_auth_patcher.stop()

        # calling base method
        super().tearDownClass()

    def setUp(self):
        # calling my base setup
        super().setUp()

        # well, updating data and set two biosample ids. Those are not
        # orphans
        animal = UIDAnimal.objects.get(pk=1)
        animal.biosample_id = "SAMEA6376980"
        animal.save()

        sample = UIDSample.objects.get(pk=1)
        sample.biosample_id = "SAMEA6376982"
        sample.save()

        # generate tocken
        self.mock_auth.return_value = Mock()
        self.mock_auth.return_value.text = generate_token()
        self.mock_auth.return_value.status_code = 200

    async def test_request(self) -> None:
        with aioresponses() as mocked:
            mocked.get(
                '{url}?filter=attr:project:IMAGE&size={size}'.format(
                    url=BIOSAMPLE_URL, size=PAGE_SIZE),
                status=200,
                body=page0)
            mocked.get(
                '{url}?filter=attr:project:IMAGE&page=1&size={size}'.format(
                    url=BIOSAMPLE_URL, size=PAGE_SIZE),
                status=200,
                body=page1)

            await check_samples()

            # get accessions
            reference = ['SAMEA6376991', 'SAMEA6376992']

            self.assertEqual(OrphanSample.objects.count(), 2)

            # check objects into UID
            for accession in reference:
                orphan = OrphanSample.objects.get(biosample_id=accession)
                orphan.status = READY

    async def test_request_with_issues(self) -> None:
        """Test a temporary issue with BioSamples reply"""

        with aioresponses() as mocked:
            mocked.get(
                '{url}?filter=attr:project:IMAGE&size={size}'.format(
                    url=BIOSAMPLE_URL, size=PAGE_SIZE),
                status=200,
                body=page0)
            mocked.get(
                '{url}?filter=attr:project:IMAGE&page=1&size={size}'.format(
                    url=BIOSAMPLE_URL, size=PAGE_SIZE),
                status=200,
                body=issue_page1)

            await check_samples()

        # no objects where tracked since issue in response
        self.assertEqual(OrphanSample.objects.count(), 0)

    async def test_request_with_html(self) -> None:
        """Test a not JSON response (HTML)"""

        with aioresponses() as mocked:
            mocked.get(
                '{url}?filter=attr:project:IMAGE&size={size}'.format(
                    url=BIOSAMPLE_URL, size=PAGE_SIZE),
                status=200,
                body=page0)
            mocked.get(
                '{url}?filter=attr:project:IMAGE&page=1&size={size}'.format(
                    url=BIOSAMPLE_URL, size=PAGE_SIZE),
                status=200,
                headers={'Content-type': 'text/html'},
                body="<html>Not a JSON</html>")

            await check_samples()

        # no objects where tracked since issue in response
        self.assertEqual(OrphanSample.objects.count(), 0)

    async def test_biosamples_down(self) -> None:
        """Test a not JSON response (HTML): BioSamples down"""

        with aioresponses() as mocked:
            mocked.get(
                '{url}?filter=attr:project:IMAGE&size={size}'.format(
                    url=BIOSAMPLE_URL, size=PAGE_SIZE),
                status=200,
                headers={'Content-type': 'text/html'},
                body="<html>Not a JSON</html>")
            mocked.get(
                '{url}?filter=attr:project:IMAGE&page=1&size={size}'.format(
                    url=BIOSAMPLE_URL, size=PAGE_SIZE),
                status=200,
                headers={'Content-type': 'text/html'},
                body="<html>Not a JSON</html>")

            with self.assertRaises(ConnectionError):
                await check_samples()

        # no objects where tracked since issue in response
        self.assertEqual(OrphanSample.objects.count(), 0)

    async def test_server_lost(self) -> None:
        """Test server disconnect error"""

        with aioresponses() as mocked:
            mocked.get(
                '{url}?filter=attr:project:IMAGE&size={size}'.format(
                    url=BIOSAMPLE_URL, size=PAGE_SIZE),
                exception=ServerDisconnectedError()
            )

            with self.assertRaises(ConnectionError):
                await check_samples()

        # no objects where tracked since issue in response
        self.assertEqual(OrphanSample.objects.count(), 0)

    async def test_already_removed_samples(self) -> None:
        """Test check samples with entries in database"""

        # create items into database. get team first
        team = ManagedTeam.objects.get(pk=1)

        sample1 = OrphanSample.objects.create(
            biosample_id='SAMEA6376991',
            name="IMAGEA000005610",
            team=team,
            status=SUBMITTED,
        )

        sample2 = OrphanSample.objects.create(
            biosample_id='SAMEA6376992',
            name="IMAGEA000005607",
            team=team,
            status=COMPLETED,
            removed=True,
            removed_at=timezone.now()
        )

        with aioresponses() as mocked:
            mocked.get(
                '{url}?filter=attr:project:IMAGE&size={size}'.format(
                    url=BIOSAMPLE_URL, size=PAGE_SIZE),
                status=200,
                body=page0)
            mocked.get(
                '{url}?filter=attr:project:IMAGE&page=1&size={size}'.format(
                    url=BIOSAMPLE_URL, size=PAGE_SIZE),
                status=200,
                body=page1)

            await check_samples()

        # test: there are the same samples in database
        self.assertEqual(OrphanSample.objects.count(), 2)

        # no changes in statuses
        self.assertEqual(sample1.status, SUBMITTED)
        self.assertEqual(sample2.status, COMPLETED)


class PurgeOrphanSampleTestCase(BioSamplesMixin, TestCase):
    fixtures = [
        'biosample/managedteam',
        'biosample/orphansample',
        'uid/dictspecie',
    ]

    def test_purge_orphan_samples(self):
        """Test biosample data conversion"""

        with open(os.path.join(DATA_PATH, "SAMEA6376982.json")) as handle:
            data = json.load(handle)

        self.mock_get.return_value = Mock()
        self.mock_get.return_value.json.return_value = data
        self.mock_get.return_value.status_code = 200

        # call my method
        samples = get_orphan_samples()

        # teams is now a generator
        self.assertIsInstance(samples, types.GeneratorType)
        samples = list(samples)

        self.assertEqual(len(samples), 2)

        sample = samples[0]
        self.assertIsInstance(sample, dict)

        sample = samples[1]
        self.assertIsInstance(sample, dict)

        # read the team from data
        team = sample['team']
        self.assertIsInstance(team, ManagedTeam)

    def test_purge_orphan_samples_not_ready(self):
        """Test not ready orphan samples"""

        # Simulate a different status
        OrphanSample.objects.update(status=SUBMITTED)
        orphan_count = sum(1 for orphan in get_orphan_samples())

        self.assertEqual(orphan_count, 0)

    def test_purge_orphan_samples_ignore(self):
        """Test ignored orphan samples"""

        # Ignoring samples gives no object
        OrphanSample.objects.update(ignore=True)
        orphan_count = sum(1 for orphan in get_orphan_samples())

        self.assertEqual(orphan_count, 0)

    def test_purge_orphan_samples_removed(self):
        """Test removed orphan samples"""

        # Ignoring samples gives no object
        OrphanSample.objects.update(removed=True)
        orphan_count = sum(1 for orphan in get_orphan_samples())

        self.assertEqual(orphan_count, 0)

    def test_purge_orphan_samples_with_limit(self):
        """Test get orphan samples with limits"""

        orphan_count = sum(1 for orphan in get_orphan_samples(limit=1))
        self.assertEqual(orphan_count, 1)

    def test_purge_orphan_private(self):
        """Test no access to a BioSamples id (already removed?)"""

        data = {
            'timestamp': int(time.time() * 1000),
            'status': 403,
            'error': 'Forbidden',
            'exception': (
                'uk.ac.ebi.biosamples.service.'
                'BioSamplesAapService$SampleNotAccessibleException'),
            'message': (
                'This sample is private and not available for browsing. '
                'If you think this is an error and/or you should have access '
                'please contact the BioSamples Helpdesk at biosamples@'
                'ebi.ac.uk'),
            'path': '/biosamples/samples/SAMEA6376982'
        }

        # override mock object
        self.mock_get.return_value = Mock()
        self.mock_get.return_value.json.return_value = data
        self.mock_get.return_value.status_code = 403

        orphan_count = sum(1 for orphan in get_orphan_samples(limit=1))
        self.assertEqual(orphan_count, 0)


1			#!/usr/bin/env python3
2			# -- coding: utf-8 --
3			"""
4			Created on Tue Jan 21 10:54:09 2020
5
6			@author: Paolo Cozzi <[email protected]>
7			"""
8
9			import os
10			import json
11			import time
12			import types
13			import asynctest
14
15			from aioresponses import aioresponses
16			from aiohttp.client_exceptions import ServerDisconnectedError
17			from unittest.mock import patch, Mock
18
19			from django.test import TestCase
20			from django.utils import timezone
21
22			from common.constants import BIOSAMPLE_URL, SUBMITTED, READY, COMPLETED
23			from uid.models import Animal as UIDAnimal, Sample as UIDSample
24
25			from ..tasks.cleanup import check_samples, get_orphan_samples, PAGE_SIZE
26			from ..models import OrphanSample, ManagedTeam
27
28			from .common import generate_token, BioSamplesMixin
29
30			# get my path
31			dir_path = os.path.dirname(os.path.realpath(__file__))
32
33			# define data path
34			DATA_PATH = os.path.join(dir_path, "data")
35
36
37			with open(os.path.join(DATA_PATH, "page_0.json")) as handle:
38			page0 = handle.read()
39
40			with open(os.path.join(DATA_PATH, "page_1.json")) as handle:
41			page1 = handle.read()
42
43			with open(os.path.join(DATA_PATH, "issue_page1.json")) as handle:
44			issue_page1 = handle.read()
45
46
47			class AsyncBioSamplesTestCase(asynctest.TestCase, TestCase):
48
49			fixtures = [
50			'biosample/managedteam',
51			'uid/animal',
52			'uid/dictbreed',
53			'uid/dictcountry',
54			'uid/dictrole',
55			'uid/dictsex',
56			'uid/dictspecie',
57			'uid/dictstage',
58			'uid/dictuberon',
59			'uid/ontology',
60			'uid/organization',
61			'uid/publication',
62			'uid/sample',
63			'uid/submission',
64			'uid/user'
65			]
66
67			@classmethod
68			def setUpClass(cls):
69			# calling my base class setup
70			super().setUpClass()
71
72			cls.mock_auth_patcher = patch('pyUSIrest.auth.requests.get')
73			cls.mock_auth = cls.mock_auth_patcher.start()
74
75			@classmethod
76			def tearDownClass(cls):
77			cls.mock_auth_patcher.stop()
78
79			# calling base method
80			super().tearDownClass()
81
82			def setUp(self):
83			# calling my base setup
84			super().setUp()
85
86			# well, updating data and set two biosample ids. Those are not
87			# orphans
88			animal = UIDAnimal.objects.get(pk=1)
89			animal.biosample_id = "SAMEA6376980"
90			animal.save()
91
92			sample = UIDSample.objects.get(pk=1)
93			sample.biosample_id = "SAMEA6376982"
94			sample.save()
95
96			# generate tocken
97			self.mock_auth.return_value = Mock()
98			self.mock_auth.return_value.text = generate_token()
99			self.mock_auth.return_value.status_code = 200
100
101			async def test_request(self) -> None:
102			with aioresponses() as mocked:
103			mocked.get(
104			'{url}?filter=attr:project:IMAGE&size={size}'.format(
105			url=BIOSAMPLE_URL, size=PAGE_SIZE),
106			status=200,
107			body=page0)
108			mocked.get(
109			'{url}?filter=attr:project:IMAGE&page=1&size={size}'.format(
110			url=BIOSAMPLE_URL, size=PAGE_SIZE),
111			status=200,
112			body=page1)
113
114			await check_samples()
115
116			# get accessions
117			reference = ['SAMEA6376991', 'SAMEA6376992']
118
119			self.assertEqual(OrphanSample.objects.count(), 2)
120
121			# check objects into UID
122			for accession in reference:
123			orphan = OrphanSample.objects.get(biosample_id=accession)
124			orphan.status = READY
125
126			async def test_request_with_issues(self) -> None:
127			"""Test a temporary issue with BioSamples reply"""
128
129			with aioresponses() as mocked:
130			mocked.get(
131			'{url}?filter=attr:project:IMAGE&size={size}'.format(
132			url=BIOSAMPLE_URL, size=PAGE_SIZE),
133			status=200,
134			body=page0)
135			mocked.get(
136			'{url}?filter=attr:project:IMAGE&page=1&size={size}'.format(
137			url=BIOSAMPLE_URL, size=PAGE_SIZE),
138			status=200,
139			body=issue_page1)
140
141			await check_samples()
142
143			# no objects where tracked since issue in response
144			self.assertEqual(OrphanSample.objects.count(), 0)
145
146			async def test_request_with_html(self) -> None:
147			"""Test a not JSON response (HTML)"""
148
149			with aioresponses() as mocked:
150			mocked.get(
151			'{url}?filter=attr:project:IMAGE&size={size}'.format(
152			url=BIOSAMPLE_URL, size=PAGE_SIZE),
153			status=200,
154			body=page0)
155			mocked.get(
156			'{url}?filter=attr:project:IMAGE&page=1&size={size}'.format(
157			url=BIOSAMPLE_URL, size=PAGE_SIZE),
158			status=200,
159			headers={'Content-type': 'text/html'},
160			body="<html>Not a JSON</html>")
161
162			await check_samples()
163
164			# no objects where tracked since issue in response
165			self.assertEqual(OrphanSample.objects.count(), 0)
166
167			async def test_biosamples_down(self) -> None:
168			"""Test a not JSON response (HTML): BioSamples down"""
169
170			with aioresponses() as mocked:
171			mocked.get(
172			'{url}?filter=attr:project:IMAGE&size={size}'.format(
173			url=BIOSAMPLE_URL, size=PAGE_SIZE),
174			status=200,
175			headers={'Content-type': 'text/html'},
176			body="<html>Not a JSON</html>")
177			mocked.get(
178			'{url}?filter=attr:project:IMAGE&page=1&size={size}'.format(
179			url=BIOSAMPLE_URL, size=PAGE_SIZE),
180			status=200,
181			headers={'Content-type': 'text/html'},
182			body="<html>Not a JSON</html>")
183
184			with self.assertRaises(ConnectionError):
185			await check_samples()
186
187			# no objects where tracked since issue in response
188			self.assertEqual(OrphanSample.objects.count(), 0)
189
190			async def test_server_lost(self) -> None:
191			"""Test server disconnect error"""
192
193			with aioresponses() as mocked:
194			mocked.get(
195			'{url}?filter=attr:project:IMAGE&size={size}'.format(
196			url=BIOSAMPLE_URL, size=PAGE_SIZE),
197			exception=ServerDisconnectedError()
198			)
199
200			with self.assertRaises(ConnectionError):
201			await check_samples()
202
203			# no objects where tracked since issue in response
204			self.assertEqual(OrphanSample.objects.count(), 0)
205
206			async def test_already_removed_samples(self) -> None:
207			"""Test check samples with entries in database"""
208
209			# create items into database. get team first
210			team = ManagedTeam.objects.get(pk=1)
211
212			sample1 = OrphanSample.objects.create(
213			biosample_id='SAMEA6376991',
214			name="IMAGEA000005610",
215			team=team,
216			status=SUBMITTED,
217			)
218
219			sample2 = OrphanSample.objects.create(
220			biosample_id='SAMEA6376992',
221			name="IMAGEA000005607",
222			team=team,
223			status=COMPLETED,
224			removed=True,
225			removed_at=timezone.now()
226			)
227
228			with aioresponses() as mocked:
229			mocked.get(
230			'{url}?filter=attr:project:IMAGE&size={size}'.format(
231			url=BIOSAMPLE_URL, size=PAGE_SIZE),
232			status=200,
233			body=page0)
234			mocked.get(
235			'{url}?filter=attr:project:IMAGE&page=1&size={size}'.format(
236			url=BIOSAMPLE_URL, size=PAGE_SIZE),
237			status=200,
238			body=page1)
239
240			await check_samples()
241
242			# test: there are the same samples in database
243			self.assertEqual(OrphanSample.objects.count(), 2)
244
245			# no changes in statuses
246			self.assertEqual(sample1.status, SUBMITTED)
247			self.assertEqual(sample2.status, COMPLETED)
248
249
250			class PurgeOrphanSampleTestCase(BioSamplesMixin, TestCase):
251			fixtures = [
252			'biosample/managedteam',
253			'biosample/orphansample',
254			'uid/dictspecie',
255			]
256
257			def test_purge_orphan_samples(self):
258			"""Test biosample data conversion"""
259
260			with open(os.path.join(DATA_PATH, "SAMEA6376982.json")) as handle:
261			data = json.load(handle)
262
263			self.mock_get.return_value = Mock()
264			self.mock_get.return_value.json.return_value = data
265			self.mock_get.return_value.status_code = 200
266
267			# call my method
268			samples = get_orphan_samples()
269
270			# teams is now a generator
271			self.assertIsInstance(samples, types.GeneratorType)
272			samples = list(samples)
273
274			self.assertEqual(len(samples), 2)
275
276			sample = samples[0]
277			self.assertIsInstance(sample, dict)
278
279			sample = samples[1]
280			self.assertIsInstance(sample, dict)
281
282			# read the team from data
283			team = sample['team']
284			self.assertIsInstance(team, ManagedTeam)
285
286			def test_purge_orphan_samples_not_ready(self):
287			"""Test not ready orphan samples"""
288
289			# Simulate a different status
290			OrphanSample.objects.update(status=SUBMITTED)
291			orphan_count = sum(1 for orphan in get_orphan_samples())
292
293			self.assertEqual(orphan_count, 0)
294
295			def test_purge_orphan_samples_ignore(self):
296			"""Test ignored orphan samples"""
297
298			# Ignoring samples gives no object
299			OrphanSample.objects.update(ignore=True)
300			orphan_count = sum(1 for orphan in get_orphan_samples())
301
302			self.assertEqual(orphan_count, 0)
303
304			def test_purge_orphan_samples_removed(self):
305			"""Test removed orphan samples"""
306
307			# Ignoring samples gives no object
308			OrphanSample.objects.update(removed=True)
309			orphan_count = sum(1 for orphan in get_orphan_samples())
310
311			self.assertEqual(orphan_count, 0)
312
313			def test_purge_orphan_samples_with_limit(self):
314			"""Test get orphan samples with limits"""
315
316			orphan_count = sum(1 for orphan in get_orphan_samples(limit=1))
317			self.assertEqual(orphan_count, 1)
318
319			def test_purge_orphan_private(self):
320			"""Test no access to a BioSamples id (already removed?)"""
321
322			data = {
323			'timestamp': int(time.time() * 1000),
324			'status': 403,
325			'error': 'Forbidden',
326			'exception': (
327			'uk.ac.ebi.biosamples.service.'
328			'BioSamplesAapService$SampleNotAccessibleException'),
329			'message': (
330			'This sample is private and not available for browsing. '
331			'If you think this is an error and/or you should have access '
332			'please contact the BioSamples Helpdesk at biosamples@'
333			'ebi.ac.uk'),
334			'path': '/biosamples/samples/SAMEA6376982'
335			}
336
337			# override mock object
338			self.mock_get.return_value = Mock()
339			self.mock_get.return_value.json.return_value = data
340			self.mock_get.return_value.status_code = 403
341
342			orphan_count = sum(1 for orphan in get_orphan_samples(limit=1))
343			self.assertEqual(orphan_count, 0)
344

cnr-ibba / IMAGE-InjectTool

Push — master ( 55dedd...16663f )

AsyncBioSamplesTestCase.test_server_lost() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like