Passed
Pull Request — master (#40)
by Paolo
01:24
created

cryoweb.helpers.check_UID()   A

Complexity

Conditions 3

Size

Total Lines 17
Code Lines 7

Duplication

Lines 0
Ratio 0 %

Importance

Changes 0
Metric Value
eloc 7
dl 0
loc 17
rs 10
c 0
b 0
f 0
cc 3
nop 1
1
#!/usr/bin/env python3
2
# -*- coding: utf-8 -*-
3
"""
4
Created on Mon May 14 10:28:39 2018
5
6
@author: Paolo Cozzi <[email protected]>
7
"""
8
9
# --- import
10
11
import logging
12
import os
13
import shlex
14
import subprocess
15
16
from decouple import AutoConfig
17
18
from django.conf import settings
19
20
from common.constants import LOADED, ERROR, MISSING, UNKNOWN
21
from common.helpers import image_timedelta
22
from image_app.helpers import get_or_create_obj, update_or_create_obj
23
from image_app.models import (
24
    Animal, DictBreed, DictCountry, DictSex, DictSpecie, Name, Sample,
25
    Submission, DictUberon)
26
from language.helpers import check_species_synonyms
27
from submissions.helpers import send_message
28
from validation.helpers import construct_validation_message
29
from validation.models import ValidationSummary
30
31
from .models import db_has_data as cryoweb_has_data
32
from .models import VAnimal, VBreedsSpecies, VTransfer, VVessels
33
34
# Get an instance of a logger
35
logger = logging.getLogger(__name__)
36
37
38
# --- check functions
39
40
41
# a function to detect if cryoweb species have synonyms or not
42
def check_species(country):
43
    """Check all cryoweb species for a synonym in a supplied language or
44
    the default one, ie: check_species(country). country is an
45
    image_app.models.DictCountry.label"""
46
47
    # get all species using view
48
    words = VBreedsSpecies.get_all_species()
49
50
    # for logging purposes
51
    database_name = settings.DATABASES['cryoweb']['NAME']
52
53
    if len(words) == 0:
54
        raise CryoWebImportError(
55
            "You have no species in %s database" % database_name)
56
57
    # debug
58
    logger.debug("Got %s species from %s" % (words, database_name))
59
60
    # check if every word as a synonym (a specie)
61
    # (And create synonyms if don't exist)
62
    return check_species_synonyms(words, country, create=True)
63
64
65
# a function specific for cryoweb import path to ensure that all required
66
# fields in UID are present. There could be a function like this in others
67
# import paths
68
def check_UID(submission):
69
    """A function to ensure that UID is valid before data upload. Specific
70
    to the module where is called from"""
71
72
    logger.debug("Checking UID")
73
74
    # check that dict sex table contains data
75
    if len(DictSex.objects.all()) == 0:
76
        raise CryoWebImportError("You have to upload DictSex data")
77
78
    # test for specie synonyms in submission language or defaul one
79
    # otherwise, fill synonym table with new terms then throw exception
80
    if not check_species(submission.gene_bank_country):
81
        raise CryoWebImportError("Some species haven't a synonym!")
82
83
    # return a status
84
    return True
85
86
87
# A class to deal with cryoweb import errors
88
class CryoWebImportError(Exception):
89
    pass
90
91
92
# --- Upload data into cryoweb database
93
def upload_cryoweb(submission_id):
94
    """Imports backup into the cryoweb db
95
96
    This function uses the container's installation of psql to import a backup
97
    file into the "cryoweb" database. The imported backup file is
98
    the last inserted into the image's table image_app_submission.
99
100
    :submission_id: the submission primary key
101
    """
102
103
    # define some useful variables
104
    database_name = settings.DATABASES['cryoweb']['NAME']
105
106
    # define a decouple config object
107
    config_dir = os.path.join(settings.BASE_DIR, 'image')
108
    config = AutoConfig(search_path=config_dir)
109
110
    # get a submission object
111
    submission = Submission.objects.get(pk=submission_id)
112
113
    # debug
114
    logger.info("Importing data into cryoweb staging area")
115
    logger.debug("Got Submission %s" % (submission))
116
117
    # If cryoweb has data, update submission message and return exception:
118
    # maybe another process is running or there is another type of problem
119
    if cryoweb_has_data():
120
        logger.error("Cryoweb has data!")
121
122
        # update submission status
123
        submission.status = ERROR
124
        submission.message = "Error in importing data: Cryoweb has data"
125
        submission.save()
126
127
        # send async message
128
        send_message(submission)
129
130
        raise CryoWebImportError("Cryoweb has data!")
131
132
    # this is the full path in docker container
133
    fullpath = submission.get_uploaded_file_path()
134
135
    # define command line
136
    cmd_line = "/usr/bin/psql -U {user} -h db {database}".format(
137
        database=database_name, user='cryoweb_insert_only')
138
139
    cmds = shlex.split(cmd_line)
140
141
    logger.debug("Executing: %s" % " ".join(cmds))
142
143
    try:
144
        result = subprocess.run(
145
            cmds,
146
            stdin=open(fullpath),
147
            stdout=subprocess.PIPE,
148
            stderr=subprocess.PIPE,
149
            check=True,
150
            env={'PGPASSWORD': config('CRYOWEB_INSERT_ONLY_PW')},
151
            encoding='utf8'
152
            )
153
154
    except Exception as exc:
155
        # save a message in database
156
        submission.status = ERROR
157
        submission.message = "Error in importing data: %s" % (str(exc))
158
        submission.save()
159
160
        # send async message
161
        send_message(submission)
162
163
        # debug
164
        logger.error("error in calling upload_cryoweb: %s" % (exc))
165
166
        return False
167
168
    n_of_statements = len(result.stdout.split("\n"))
169
    logger.debug("%s statement executed" % n_of_statements)
170
171
    if len(result.stderr) > 0:
172
        for line in result.stderr.split("\n"):
173
            logger.error(line)
174
175
    logger.info("{filename} uploaded into {database}".format(
176
        filename=submission.uploaded_file.name, database=database_name))
177
178
    return True
179
180
181
# --- Upload data from cryoweb to UID
182
183
184
def fill_uid_breeds(submission):
185
    """Fill UID DictBreed model. Require a submission instance"""
186
187
    logger.info("fill_uid_breeds() started")
188
189
    # get submission language
190
    language = submission.gene_bank_country.label
191
192
    for v_breed_specie in VBreedsSpecies.objects.all():
193
        # get specie. Since I need a dictionary tables, DictSpecie is
194
        # already filled
195
        specie = DictSpecie.get_by_synonym(
196
            synonym=v_breed_specie.ext_species,
197
            language=language)
198
199
        # get country for breeds. Ideally will be the same of submission,
200
        # since the Italian cryoweb is supposed to contains italian breeds.
201
        # however, it could be possible to store data from other contries
202
        country = get_or_create_obj(
203
            DictCountry,
204
            label=v_breed_specie.efabis_country)
205
206
        # create breed obj if necessary
207
        get_or_create_obj(
208
            DictBreed,
209
            supplied_breed=v_breed_specie.efabis_mcname,
210
            specie=specie,
211
            country=country)
212
213
    logger.info("fill_uid_breeds() completed")
214
215
216
def fill_uid_names(submission):
217
    """Read VTransfer Views and fill name table"""
218
219
    # debug
220
    logger.info("called fill_uid_names()")
221
222
    # get all Vtransfer object
223
    for v_tranfer in VTransfer.objects.all():
224
        # no name manipulation. If two objects are indentical, there's no
225
        # duplicates.
226
        # HINT: The ramon example will be a issue in validation step
227
        get_or_create_obj(
228
            Name,
229
            name=v_tranfer.get_fullname(),
230
            submission=submission,
231
            owner=submission.owner)
232
233
    logger.info("fill_uid_names() completed")
234
235
236
def fill_uid_animals(submission):
237
    """Helper function to fill animal data in UID animal table"""
238
239
    # debug
240
    logger.info("called fill_uid_animals()")
241
242
    # get submission language
243
    language = submission.gene_bank_country.label
244
245
    # get male and female DictSex objects from database
246
    male = DictSex.objects.get(label="male")
247
    female = DictSex.objects.get(label="female")
248
249
    # cycle over animals
250
    for v_animal in VAnimal.objects.all():
251
        # get specie translated by dictionary
252
        specie = DictSpecie.get_by_synonym(
253
            synonym=v_animal.ext_species,
254
            language=language)
255
256
        # get breed name and country through VBreedsSpecies model
257
        efabis_mcname = v_animal.efabis_mcname
258
        efabis_country = v_animal.efabis_country
259
260
        # get a country object
261
        country = DictCountry.objects.get(label=efabis_country)
262
263
        # a breed could be specie/country specific
264
        breed = DictBreed.objects.get(
265
            supplied_breed=efabis_mcname,
266
            specie=specie,
267
            country=country)
268
269
        logger.debug("Selected breed is %s" % (breed))
270
271
        # get name for this animal and for mother and father
272
        logger.debug("Getting %s as my name" % (v_animal.ext_animal))
273
        name = Name.objects.get(
274
            name=v_animal.ext_animal, submission=submission)
275
276
        logger.debug("Getting %s as father" % (v_animal.ext_sire))
277
        father = Name.objects.get(
278
            name=v_animal.ext_sire, submission=submission)
279
280
        logger.debug("Getting %s as mother" % (v_animal.ext_dam))
281
        mother = Name.objects.get(
282
            name=v_animal.ext_dam, submission=submission)
283
284
        # determine sex. Check for values
285
        if v_animal.ext_sex == 'm':
286
            sex = male
287
288
        elif v_animal.ext_sex == 'f':
289
            sex = female
290
291
        else:
292
            raise CryoWebImportError(
293
                "Unknown sex '%s' for '%s'" % (v_animal.ext_sex, v_animal))
294
295
        # checking accuracy
296
        accuracy = MISSING
297
298
        if v_animal.latitude and v_animal.longitude:
299
            accuracy = UNKNOWN
300
301
        # create a new object. Using defaults to avoid collisions when
302
        # updating data
303
        defaults = {
304
            'alternative_id': v_animal.db_animal,
305
            'breed': breed,
306
            'sex': sex,
307
            'father': father,
308
            'mother': mother,
309
            'birth_date': v_animal.birth_dt,
310
            'birth_location_latitude': v_animal.latitude,
311
            'birth_location_longitude': v_animal.longitude,
312
            'birth_location_accuracy': accuracy,
313
            'description': v_animal.comment,
314
            'owner': submission.owner
315
        }
316
317
        # Upate or create animal obj
318
        update_or_create_obj(
319
            Animal,
320
            name=name,
321
            defaults=defaults)
322
323
    # create a validation summary object and set all_count
324
    validation_summary = get_or_create_obj(
325
        ValidationSummary,
326
        submission=submission,
327
        type="animal")
328
329
    # reset counts
330
    validation_summary.reset_all_count()
331
332
    # debug
333
    logger.info("fill_uid_animals() completed")
334
335
336
def fill_uid_samples(submission):
337
    """Helper function to fill animal data in UID animal table"""
338
339
    # debug
340
    logger.info("called fill_uid_samples()")
341
342
    for v_vessel in VVessels.objects.all():
343
        # get name for this sample. Need to insert it
344
        name = get_or_create_obj(
345
            Name,
346
            name=v_vessel.ext_vessel,
347
            submission=submission,
348
            owner=submission.owner)
349
350
        # get animal object using name
351
        animal = Animal.objects.get(
352
            name__name=v_vessel.ext_animal,
353
            name__submission=submission)
354
355
        # get a organism part. Organism parts need to be in lowercases
356
        organism_part = get_or_create_obj(
357
            DictUberon,
358
            label=v_vessel.get_organism_part().lower()
359
        )
360
361
        # get a v_animal instance to get access to animal birth date
362
        v_animal = VAnimal.objects.get(db_animal=v_vessel.db_animal)
363
364
        # derive animal age at collection. THis function deals with NULL valies
365
        animal_age_at_collection, time_units = image_timedelta(
366
            v_vessel.production_dt, v_animal.birth_dt)
367
368
        # create a new object. Using defaults to avoid collisions when
369
        # updating data
370
        defaults = {
371
            'alternative_id': v_vessel.db_vessel,
372
            'collection_date': v_vessel.production_dt,
373
            # 'protocol': v_vessel.get_protocol_name(),
374
            'organism_part': organism_part,
375
            'animal': animal,
376
            'description': v_vessel.comment,
377
            'owner': submission.owner,
378
            'animal_age_at_collection': animal_age_at_collection,
379
            'animal_age_at_collection_units': time_units,
380
            # 'storage': v_vessel.ext_vessel_type,
381
        }
382
383
        update_or_create_obj(
384
            Sample,
385
            name=name,
386
            defaults=defaults)
387
388
    # create a validation summary object and set all_count
389
    validation_summary = get_or_create_obj(
390
        ValidationSummary,
391
        submission=submission,
392
        type="sample")
393
394
    # reset counts
395
    validation_summary.reset_all_count()
396
397
    # debug
398
    logger.info("fill_uid_samples() completed")
399
400
401
def cryoweb_import(submission):
402
    """Import data from cryoweb stage database into UID
403
404
    :submission: a submission instance
405
    """
406
407
    # debug
408
    logger.info("Importing from cryoweb staging area")
409
410
    try:
411
        # check UID status. get an exception if database is not initialized
412
        check_UID(submission)
413
414
        # BREEDS
415
        fill_uid_breeds(submission)
416
417
        # NAME
418
        fill_uid_names(submission)
419
420
        # ANIMALS
421
        fill_uid_animals(submission)
422
423
        # SAMPLES
424
        fill_uid_samples(submission)
425
426
    except Exception as exc:
427
        # save a message in database
428
        submission.status = ERROR
429
        submission.message = "Error in importing data: %s" % (str(exc))
430
        submission.save()
431
432
        # send async message
433
        send_message(submission)
434
435
        # debug
436
        logger.error("error in importing from cryoweb: %s" % (exc))
437
        logger.exception(exc)
438
439
        return False
440
441
    else:
442
        message = "Cryoweb import completed for submission: %s" % (
443
            submission.id)
444
445
        submission.message = message
446
        submission.status = LOADED
447
        submission.save()
448
449
        send_message(
450
            submission,
451
            validation_message=construct_validation_message(submission))
452
453
    logger.info("Import from staging area is complete")
454
455
    return True
456