biosample.tests.test_async_biosample.AsyncBioSamplesTestCase.check_samples() - Code Metrics - Inspection of ":bug: solve issues with SearchOrphanTask" - cnr-ibba/IMAGE-InjectTool - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — devel (#121)

by Paolo

created 2023-06-08 12:29 UTC

AsyncBioSamplesTestCase.check_samples() A

↳ Parent: biosample.tests.test_async_biosample

Complexity

Conditions

Size

Total Lines	20
Code Lines	19

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
eloc	19
dl	0
loc	20
rs	9.45
c	0
b	0
f	0
cc	3
nop	1

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Jan 21 10:54:09 2020

@author: Paolo Cozzi <[email protected]>
"""

import os
import json
import time
import types
import asynctest

from aioresponses import aioresponses
from aiohttp.client_exceptions import ServerDisconnectedError
from unittest.mock import patch, Mock

from django.test import TestCase
from django.utils import timezone

from common.constants import SUBMITTED, READY, COMPLETED
from uid.models import Animal as UIDAnimal, Sample as UIDSample

from ..tasks.cleanup import (
    check_samples, get_orphan_samples, PAGE_SIZE, BIOSAMPLE_ACCESSION_ENDPOINT,
    BIOSAMPLE_SAMPLE_ENDPOINT)
from ..models import OrphanSample, ManagedTeam

from .common import generate_token, BioSamplesMixin

# get my path
dir_path = os.path.dirname(os.path.realpath(__file__))

# define data path
DATA_PATH = os.path.join(dir_path, "data")


with open(os.path.join(DATA_PATH, "page_0.json")) as handle:
    page0 = handle.read()

with open(os.path.join(DATA_PATH, "page_1.json")) as handle:
    page1 = handle.read()

with open(os.path.join(DATA_PATH, "issue_page1.json")) as handle:
    issue_page1 = handle.read()

mocked_accessions = {}

for accession in ["SAMEA6376679", "SAMEA6376682", "SAMEA6376980",
                  "SAMEA6376982", "SAMEA6376991", "SAMEA6376992"]:
    with open(os.path.join(DATA_PATH, f"{accession}.json")) as handle:
        mocked_accessions[accession] = handle.read()


class AsyncBioSamplesTestCase(asynctest.TestCase, TestCase):

    fixtures = [
        'biosample/managedteam',
        'uid/animal',
        'uid/dictbreed',
        'uid/dictcountry',
        'uid/dictrole',
        'uid/dictsex',
        'uid/dictspecie',
        'uid/dictstage',
        'uid/dictuberon',
        'uid/ontology',
        'uid/organization',
        'uid/publication',
        'uid/sample',
        'uid/submission',
        'uid/user'
    ]

    @classmethod
    def setUpClass(cls):
        # calling my base class setup
        super().setUpClass()

        cls.mock_auth_patcher = patch('pyUSIrest.auth.requests.get')
        cls.mock_auth = cls.mock_auth_patcher.start()

    @classmethod
    def tearDownClass(cls):
        cls.mock_auth_patcher.stop()

        # calling base method
        super().tearDownClass()

    def setUp(self):
        # calling my base setup
        super().setUp()

        # well, updating data and set two biosample ids. Those are not
        # orphans
        animal = UIDAnimal.objects.get(pk=1)
        animal.biosample_id = "SAMEA6376980"
        animal.save()

        sample = UIDSample.objects.get(pk=1)
        sample.biosample_id = "SAMEA6376982"
        sample.save()

        # generate tocken
        self.mock_auth.return_value = Mock()
        self.mock_auth.return_value.text = generate_token()
        self.mock_auth.return_value.status_code = 200

    async def check_samples(self):
        with aioresponses() as mocked:
            mocked.get(
                '{url}?filter=attr:project:IMAGE&size={size}'.format(
                    url=BIOSAMPLE_ACCESSION_ENDPOINT, size=PAGE_SIZE),
                status=200,
                body=page0)
            mocked.get(
                '{url}?filter=attr:project:IMAGE&page=1&size={size}'.format(
                    url=BIOSAMPLE_ACCESSION_ENDPOINT, size=PAGE_SIZE),
                status=200,
                body=page1)
            for accession, body in mocked_accessions.items():
                mocked.get(
                    '{url}/{accession}'.format(
                        url=BIOSAMPLE_SAMPLE_ENDPOINT, accession=accession),
                    status=200,
                    body=body)

            await check_samples()

    async def test_request(self) -> None:
        await self.check_samples()

        # get accessions
        reference = ['SAMEA6376991', 'SAMEA6376992']

        self.assertEqual(OrphanSample.objects.count(), 2)

        # check objects into UID
        for accession in reference:
            orphan = OrphanSample.objects.get(biosample_id=accession)
            orphan.status = READY

    async def test_request_with_issues(self) -> None:
        """Test a temporary issue with BioSamples reply"""

        with aioresponses() as mocked:
            mocked.get(
                '{url}?filter=attr:project:IMAGE&size={size}'.format(
                    url=BIOSAMPLE_ACCESSION_ENDPOINT, size=PAGE_SIZE),
                status=200,
                body=page0)
            mocked.get(
                '{url}?filter=attr:project:IMAGE&page=1&size={size}'.format(
                    url=BIOSAMPLE_ACCESSION_ENDPOINT, size=PAGE_SIZE),
                status=200,
                body=issue_page1)
            for accession, body in mocked_accessions.items():
                mocked.get(
                    '{url}/{accession}'.format(
                        url=BIOSAMPLE_SAMPLE_ENDPOINT, accession=accession),
                    status=200,
                    body=body)

            await check_samples()

        # no objects where tracked since issue in response
        self.assertEqual(OrphanSample.objects.count(), 0)

    async def test_request_with_html(self) -> None:
        """Test a not JSON response (HTML)"""

        with aioresponses() as mocked:
            mocked.get(
                '{url}?filter=attr:project:IMAGE&size={size}'.format(
                    url=BIOSAMPLE_ACCESSION_ENDPOINT, size=PAGE_SIZE),
                status=200,
                body=page0)
            mocked.get(
                '{url}?filter=attr:project:IMAGE&page=1&size={size}'.format(
                    url=BIOSAMPLE_ACCESSION_ENDPOINT, size=PAGE_SIZE),
                status=200,
                headers={'Content-type': 'text/html'},
                body="<html>Not a JSON</html>")
            for accession, body in mocked_accessions.items():
                mocked.get(
                    '{url}/{accession}'.format(
                        url=BIOSAMPLE_SAMPLE_ENDPOINT, accession=accession),
                    status=200,
                    body=body)

            await check_samples()

        # no objects where tracked since issue in response
        self.assertEqual(OrphanSample.objects.count(), 0)

    async def test_biosamples_down(self) -> None:
        """Test a not JSON response (HTML): BioSamples down"""

        with aioresponses() as mocked:
            mocked.get(
                '{url}?filter=attr:project:IMAGE&size={size}'.format(
                    url=BIOSAMPLE_ACCESSION_ENDPOINT, size=PAGE_SIZE),
                status=200,
                headers={'Content-type': 'text/html'},
                body="<html>Not a JSON</html>")
            mocked.get(
                '{url}?filter=attr:project:IMAGE&page=1&size={size}'.format(
                    url=BIOSAMPLE_ACCESSION_ENDPOINT, size=PAGE_SIZE),
                status=200,
                headers={'Content-type': 'text/html'},
                body="<html>Not a JSON</html>")

            with self.assertRaises(ConnectionError):
                await check_samples()

        # no objects where tracked since issue in response
        self.assertEqual(OrphanSample.objects.count(), 0)

    async def test_server_lost(self) -> None:
        """Test server disconnect error"""

        with aioresponses() as mocked:
            mocked.get(
                '{url}?filter=attr:project:IMAGE&size={size}'.format(
                    url=BIOSAMPLE_ACCESSION_ENDPOINT, size=PAGE_SIZE),
                exception=ServerDisconnectedError()
            )

            with self.assertRaises(ConnectionError):
                await check_samples()

        # no objects where tracked since issue in response
        self.assertEqual(OrphanSample.objects.count(), 0)

    async def test_already_removed_samples(self) -> None:
        """Test check samples with entries in database"""

        # create items into database. get team first
        team = ManagedTeam.objects.get(pk=1)

        sample1 = OrphanSample.objects.create(
            biosample_id='SAMEA6376991',
            name="IMAGEA000005610",
            team=team,
            status=SUBMITTED,
        )

        sample2 = OrphanSample.objects.create(
            biosample_id='SAMEA6376992',
            name="IMAGEA000005607",
            team=team,
            status=COMPLETED,
            removed=True,
            removed_at=timezone.now()
        )

        await self.check_samples()

        # test: there are the same samples in database
        self.assertEqual(OrphanSample.objects.count(), 2)

        # no changes in statuses
        self.assertEqual(sample1.status, SUBMITTED)
        self.assertEqual(sample2.status, COMPLETED)


class PurgeOrphanSampleTestCase(BioSamplesMixin, TestCase):
    fixtures = [
        'biosample/managedteam',
        'biosample/orphansample',
        'uid/dictspecie',
    ]

    def test_purge_orphan_samples(self):
        """Test biosample data conversion"""

        with open(os.path.join(DATA_PATH, "SAMEA6376982.json")) as handle:
            data = json.load(handle)

        self.mock_get.return_value = Mock()
        self.mock_get.return_value.json.return_value = data
        self.mock_get.return_value.status_code = 200

        # call my method
        samples = get_orphan_samples()

        # teams is now a generator
        self.assertIsInstance(samples, types.GeneratorType)
        samples = list(samples)

        self.assertEqual(len(samples), 2)

        sample = samples[0]
        self.assertIsInstance(sample, dict)

        sample = samples[1]
        self.assertIsInstance(sample, dict)

        # read the team from data
        team = sample['team']
        self.assertIsInstance(team, ManagedTeam)

    def test_purge_orphan_samples_not_ready(self):
        """Test not ready orphan samples"""

        # Simulate a different status
        OrphanSample.objects.update(status=SUBMITTED)
        orphan_count = sum(1 for orphan in get_orphan_samples())

        self.assertEqual(orphan_count, 0)

    def test_purge_orphan_samples_ignore(self):
        """Test ignored orphan samples"""

        # Ignoring samples gives no object
        OrphanSample.objects.update(ignore=True)
        orphan_count = sum(1 for orphan in get_orphan_samples())

        self.assertEqual(orphan_count, 0)

    def test_purge_orphan_samples_removed(self):
        """Test removed orphan samples"""

        # Ignoring samples gives no object
        OrphanSample.objects.update(removed=True)
        orphan_count = sum(1 for orphan in get_orphan_samples())

        self.assertEqual(orphan_count, 0)

    def test_purge_orphan_samples_with_limit(self):
        """Test get orphan samples with limits"""

        orphan_count = sum(1 for orphan in get_orphan_samples(limit=1))
        self.assertEqual(orphan_count, 1)

    def test_purge_orphan_private(self):
        """Test no access to a BioSamples id (already removed?)"""

        data = {
            'timestamp': int(time.time() * 1000),
            'status': 403,
            'error': 'Forbidden',
            'exception': (
                'uk.ac.ebi.biosamples.service.'
                'BioSamplesAapService$SampleNotAccessibleException'),
            'message': (
                'This sample is private and not available for browsing. '
                'If you think this is an error and/or you should have access '
                'please contact the BioSamples Helpdesk at biosamples@'
                'ebi.ac.uk'),
            'path': '/biosamples/samples/SAMEA6376982'
        }

        # override mock object
        self.mock_get.return_value = Mock()
        self.mock_get.return_value.json.return_value = data
        self.mock_get.return_value.status_code = 403

        orphan_count = sum(1 for orphan in get_orphan_samples(limit=1))
        self.assertEqual(orphan_count, 0)


1			#!/usr/bin/env python3
2			# -- coding: utf-8 --
3			"""
4			Created on Tue Jan 21 10:54:09 2020
5
6			@author: Paolo Cozzi <[email protected]>
7			"""
8
9			import os
10			import json
11			import time
12			import types
13			import asynctest
14
15			from aioresponses import aioresponses
16			from aiohttp.client_exceptions import ServerDisconnectedError
17			from unittest.mock import patch, Mock
18
19			from django.test import TestCase
20			from django.utils import timezone
21
22			from common.constants import SUBMITTED, READY, COMPLETED
23			from uid.models import Animal as UIDAnimal, Sample as UIDSample
24
25			from ..tasks.cleanup import (
26			check_samples, get_orphan_samples, PAGE_SIZE, BIOSAMPLE_ACCESSION_ENDPOINT,
27			BIOSAMPLE_SAMPLE_ENDPOINT)
28			from ..models import OrphanSample, ManagedTeam
29
30			from .common import generate_token, BioSamplesMixin
31
32			# get my path
33			dir_path = os.path.dirname(os.path.realpath(__file__))
34
35			# define data path
36			DATA_PATH = os.path.join(dir_path, "data")
37
38
39			with open(os.path.join(DATA_PATH, "page_0.json")) as handle:
40			page0 = handle.read()
41
42			with open(os.path.join(DATA_PATH, "page_1.json")) as handle:
43			page1 = handle.read()
44
45			with open(os.path.join(DATA_PATH, "issue_page1.json")) as handle:
46			issue_page1 = handle.read()
47
48			mocked_accessions = {}
49
50			for accession in ["SAMEA6376679", "SAMEA6376682", "SAMEA6376980",
51			"SAMEA6376982", "SAMEA6376991", "SAMEA6376992"]:
52			with open(os.path.join(DATA_PATH, f"{accession}.json")) as handle:
53			mocked_accessions[accession] = handle.read()
54
55
56			class AsyncBioSamplesTestCase(asynctest.TestCase, TestCase):
57
58			fixtures = [
59			'biosample/managedteam',
60			'uid/animal',
61			'uid/dictbreed',
62			'uid/dictcountry',
63			'uid/dictrole',
64			'uid/dictsex',
65			'uid/dictspecie',
66			'uid/dictstage',
67			'uid/dictuberon',
68			'uid/ontology',
69			'uid/organization',
70			'uid/publication',
71			'uid/sample',
72			'uid/submission',
73			'uid/user'
74			]
75
76			@classmethod
77			def setUpClass(cls):
78			# calling my base class setup
79			super().setUpClass()
80
81			cls.mock_auth_patcher = patch('pyUSIrest.auth.requests.get')
82			cls.mock_auth = cls.mock_auth_patcher.start()
83
84			@classmethod
85			def tearDownClass(cls):
86			cls.mock_auth_patcher.stop()
87
88			# calling base method
89			super().tearDownClass()
90
91			def setUp(self):
92			# calling my base setup
93			super().setUp()
94
95			# well, updating data and set two biosample ids. Those are not
96			# orphans
97			animal = UIDAnimal.objects.get(pk=1)
98			animal.biosample_id = "SAMEA6376980"
99			animal.save()
100
101			sample = UIDSample.objects.get(pk=1)
102			sample.biosample_id = "SAMEA6376982"
103			sample.save()
104
105			# generate tocken
106			self.mock_auth.return_value = Mock()
107			self.mock_auth.return_value.text = generate_token()
108			self.mock_auth.return_value.status_code = 200
109
110			async def check_samples(self):
111			with aioresponses() as mocked:
112			mocked.get(
113			'{url}?filter=attr:project:IMAGE&size={size}'.format(
114			url=BIOSAMPLE_ACCESSION_ENDPOINT, size=PAGE_SIZE),
115			status=200,
116			body=page0)
117			mocked.get(
118			'{url}?filter=attr:project:IMAGE&page=1&size={size}'.format(
119			url=BIOSAMPLE_ACCESSION_ENDPOINT, size=PAGE_SIZE),
120			status=200,
121			body=page1)
122			for accession, body in mocked_accessions.items():
123			mocked.get(
124			'{url}/{accession}'.format(
125			url=BIOSAMPLE_SAMPLE_ENDPOINT, accession=accession),
126			status=200,
127			body=body)
128
129			await check_samples()
130
131			async def test_request(self) -> None:
132			await self.check_samples()
133
134			# get accessions
135			reference = ['SAMEA6376991', 'SAMEA6376992']
136
137			self.assertEqual(OrphanSample.objects.count(), 2)
138
139			# check objects into UID
140			for accession in reference:
141			orphan = OrphanSample.objects.get(biosample_id=accession)
142			orphan.status = READY
143
144			async def test_request_with_issues(self) -> None:
145			"""Test a temporary issue with BioSamples reply"""
146
147			with aioresponses() as mocked:
148			mocked.get(
149			'{url}?filter=attr:project:IMAGE&size={size}'.format(
150			url=BIOSAMPLE_ACCESSION_ENDPOINT, size=PAGE_SIZE),
151			status=200,
152			body=page0)
153			mocked.get(
154			'{url}?filter=attr:project:IMAGE&page=1&size={size}'.format(
155			url=BIOSAMPLE_ACCESSION_ENDPOINT, size=PAGE_SIZE),
156			status=200,
157			body=issue_page1)
158			for accession, body in mocked_accessions.items():
159			mocked.get(
160			'{url}/{accession}'.format(
161			url=BIOSAMPLE_SAMPLE_ENDPOINT, accession=accession),
162			status=200,
163			body=body)
164
165			await check_samples()
166
167			# no objects where tracked since issue in response
168			self.assertEqual(OrphanSample.objects.count(), 0)
169
170			async def test_request_with_html(self) -> None:
171			"""Test a not JSON response (HTML)"""
172
173			with aioresponses() as mocked:
174			mocked.get(
175			'{url}?filter=attr:project:IMAGE&size={size}'.format(
176			url=BIOSAMPLE_ACCESSION_ENDPOINT, size=PAGE_SIZE),
177			status=200,
178			body=page0)
179			mocked.get(
180			'{url}?filter=attr:project:IMAGE&page=1&size={size}'.format(
181			url=BIOSAMPLE_ACCESSION_ENDPOINT, size=PAGE_SIZE),
182			status=200,
183			headers={'Content-type': 'text/html'},
184			body="<html>Not a JSON</html>")
185			for accession, body in mocked_accessions.items():
186			mocked.get(
187			'{url}/{accession}'.format(
188			url=BIOSAMPLE_SAMPLE_ENDPOINT, accession=accession),
189			status=200,
190			body=body)
191
192			await check_samples()
193
194			# no objects where tracked since issue in response
195			self.assertEqual(OrphanSample.objects.count(), 0)
196
197			async def test_biosamples_down(self) -> None:
198			"""Test a not JSON response (HTML): BioSamples down"""
199
200			with aioresponses() as mocked:
201			mocked.get(
202			'{url}?filter=attr:project:IMAGE&size={size}'.format(
203			url=BIOSAMPLE_ACCESSION_ENDPOINT, size=PAGE_SIZE),
204			status=200,
205			headers={'Content-type': 'text/html'},
206			body="<html>Not a JSON</html>")
207			mocked.get(
208			'{url}?filter=attr:project:IMAGE&page=1&size={size}'.format(
209			url=BIOSAMPLE_ACCESSION_ENDPOINT, size=PAGE_SIZE),
210			status=200,
211			headers={'Content-type': 'text/html'},
212			body="<html>Not a JSON</html>")
213
214			with self.assertRaises(ConnectionError):
215			await check_samples()
216
217			# no objects where tracked since issue in response
218			self.assertEqual(OrphanSample.objects.count(), 0)
219
220			async def test_server_lost(self) -> None:
221			"""Test server disconnect error"""
222
223			with aioresponses() as mocked:
224			mocked.get(
225			'{url}?filter=attr:project:IMAGE&size={size}'.format(
226			url=BIOSAMPLE_ACCESSION_ENDPOINT, size=PAGE_SIZE),
227			exception=ServerDisconnectedError()
228			)
229
230			with self.assertRaises(ConnectionError):
231			await check_samples()
232
233			# no objects where tracked since issue in response
234			self.assertEqual(OrphanSample.objects.count(), 0)
235
236			async def test_already_removed_samples(self) -> None:
237			"""Test check samples with entries in database"""
238
239			# create items into database. get team first
240			team = ManagedTeam.objects.get(pk=1)
241
242			sample1 = OrphanSample.objects.create(
243			biosample_id='SAMEA6376991',
244			name="IMAGEA000005610",
245			team=team,
246			status=SUBMITTED,
247			)
248
249			sample2 = OrphanSample.objects.create(
250			biosample_id='SAMEA6376992',
251			name="IMAGEA000005607",
252			team=team,
253			status=COMPLETED,
254			removed=True,
255			removed_at=timezone.now()
256			)
257
258			await self.check_samples()
259
260			# test: there are the same samples in database
261			self.assertEqual(OrphanSample.objects.count(), 2)
262
263			# no changes in statuses
264			self.assertEqual(sample1.status, SUBMITTED)
265			self.assertEqual(sample2.status, COMPLETED)
266
267
268			class PurgeOrphanSampleTestCase(BioSamplesMixin, TestCase):
269			fixtures = [
270			'biosample/managedteam',
271			'biosample/orphansample',
272			'uid/dictspecie',
273			]
274
275			def test_purge_orphan_samples(self):
276			"""Test biosample data conversion"""
277
278			with open(os.path.join(DATA_PATH, "SAMEA6376982.json")) as handle:
279			data = json.load(handle)
280
281			self.mock_get.return_value = Mock()
282			self.mock_get.return_value.json.return_value = data
283			self.mock_get.return_value.status_code = 200
284
285			# call my method
286			samples = get_orphan_samples()
287
288			# teams is now a generator
289			self.assertIsInstance(samples, types.GeneratorType)
290			samples = list(samples)
291
292			self.assertEqual(len(samples), 2)
293
294			sample = samples[0]
295			self.assertIsInstance(sample, dict)
296
297			sample = samples[1]
298			self.assertIsInstance(sample, dict)
299
300			# read the team from data
301			team = sample['team']
302			self.assertIsInstance(team, ManagedTeam)
303
304			def test_purge_orphan_samples_not_ready(self):
305			"""Test not ready orphan samples"""
306
307			# Simulate a different status
308			OrphanSample.objects.update(status=SUBMITTED)
309			orphan_count = sum(1 for orphan in get_orphan_samples())
310
311			self.assertEqual(orphan_count, 0)
312
313			def test_purge_orphan_samples_ignore(self):
314			"""Test ignored orphan samples"""
315
316			# Ignoring samples gives no object
317			OrphanSample.objects.update(ignore=True)
318			orphan_count = sum(1 for orphan in get_orphan_samples())
319
320			self.assertEqual(orphan_count, 0)
321
322			def test_purge_orphan_samples_removed(self):
323			"""Test removed orphan samples"""
324
325			# Ignoring samples gives no object
326			OrphanSample.objects.update(removed=True)
327			orphan_count = sum(1 for orphan in get_orphan_samples())
328
329			self.assertEqual(orphan_count, 0)
330
331			def test_purge_orphan_samples_with_limit(self):
332			"""Test get orphan samples with limits"""
333
334			orphan_count = sum(1 for orphan in get_orphan_samples(limit=1))
335			self.assertEqual(orphan_count, 1)
336
337			def test_purge_orphan_private(self):
338			"""Test no access to a BioSamples id (already removed?)"""
339
340			data = {
341			'timestamp': int(time.time() * 1000),
342			'status': 403,
343			'error': 'Forbidden',
344			'exception': (
345			'uk.ac.ebi.biosamples.service.'
346			'BioSamplesAapService$SampleNotAccessibleException'),
347			'message': (
348			'This sample is private and not available for browsing. '
349			'If you think this is an error and/or you should have access '
350			'please contact the BioSamples Helpdesk at biosamples@'
351			'ebi.ac.uk'),
352			'path': '/biosamples/samples/SAMEA6376982'
353			}
354
355			# override mock object
356			self.mock_get.return_value = Mock()
357			self.mock_get.return_value.json.return_value = data
358			self.mock_get.return_value.status_code = 403
359
360			orphan_count = sum(1 for orphan in get_orphan_samples(limit=1))
361			self.assertEqual(orphan_count, 0)
362

cnr-ibba / IMAGE-InjectTool

Pull Request — devel (#121)

AsyncBioSamplesTestCase.check_samples() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like