cryoweb.helpers.fill_uid_breeds() - Code Metrics - Inspection of "last requirements of Validation Summary Views issu..." - cnr-ibba/IMAGE-InjectTool - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — master (#35)

by Paolo

created 2019-06-20 15:45 UTC

cryoweb.helpers.fill_uid_breeds() A

↳ Parent: cryoweb.helpers

Complexity

Conditions

Size

Total Lines	42
Code Lines	20

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
eloc	20
dl	0
loc	42
rs	9.4
c	0
b	0
f	0
cc	4
nop	1

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon May 14 10:28:39 2018

@author: Paolo Cozzi <[email protected]>
"""

# --- import

import logging
import os
import shlex
import subprocess
import asyncio

from decouple import AutoConfig

from django.conf import settings

from common.constants import LOADED, ERROR, MISSING, UNKNOWN, STATUSES
from common.helpers import image_timedelta, send_message_to_websocket
from image_app.models import (
    Animal, DictBreed, DictCountry, DictSex, DictSpecie, Name, Sample,
    Submission, DictUberon)
from language.helpers import check_species_synonyms
from validation.helpers import construct_validation_message
from validation.models import ValidationSummary

from .models import db_has_data as cryoweb_has_data
from .models import VAnimal, VBreedsSpecies, VTransfer, VVessels

# Get an instance of a logger
logger = logging.getLogger(__name__)


# --- check functions


# a function to detect if cryoweb species have synonyms or not
def check_species(country):
    """Check all cryoweb species for a synonym in a supplied language or
    the default one, ie: check_species(country). country is an
    image_app.models.DictCountry.label"""

    # get all species using view
    words = VBreedsSpecies.get_all_species()

    # for logging purposes
    database_name = settings.DATABASES['cryoweb']['NAME']

    if len(words) == 0:
        raise CryoWebImportError(
            "You have no species in %s database" % database_name)

    # debug
    logger.debug("Got %s species from %s" % (words, database_name))

    # check if every word as a synonym (a specie)
    # (And create synonyms if don't exist)
    return check_species_synonyms(words, country, create=True)


# a function specific for cryoweb import path to ensure that all required
# fields in UID are present. There could be a function like this in others
# import paths
def check_UID(submission):
    """A function to ensure that UID is valid before data upload. Specific
    to the module where is called from"""

    logger.debug("Checking UID")

    # check that dict sex table contains data
    if len(DictSex.objects.all()) == 0:
        raise CryoWebImportError("You have to upload DictSex data")

    # test for specie synonyms in submission language or defaul one
    # otherwise, fill synonym table with new terms then throw exception
    if not check_species(submission.gene_bank_country):
        raise CryoWebImportError("Some species haven't a synonym!")

    # return a status
    return True


# A class to deal with cryoweb import errors
class CryoWebImportError(Exception):
    pass


def send_message(submission_obj, send_validation=False):

    """
    Update submission.status and submission message using django
    channels

    Args:
        submission_obj (image_app.models.Submission): an UID submission
        object
        send_validation (bool): send validation message or not
    """

    # define a message to send
    message = {
        'message': STATUSES.get_value_display(submission_obj.status),
        'notification_message': submission_obj.message,
    }

    # if validation message is needed, add to the final message
    if send_validation:
        message['validation_message'] = construct_validation_message(
            submission_obj)

    # now send the message to its submission
    asyncio.get_event_loop().run_until_complete(
        send_message_to_websocket(
            message,
            submission_obj.pk
        )
    )


# --- Upload data into cryoweb database
def upload_cryoweb(submission_id):
    """Imports backup into the cryoweb db

    This function uses the container's installation of psql to import a backup
    file into the "cryoweb" database. The imported backup file is
    the last inserted into the image's table image_app_submission.

    :submission_id: the submission primary key
    """

    # define some useful variables
    database_name = settings.DATABASES['cryoweb']['NAME']

    # define a decouple config object
    config_dir = os.path.join(settings.BASE_DIR, 'image')
    config = AutoConfig(search_path=config_dir)

    # get a submission object
    submission = Submission.objects.get(pk=submission_id)

    # debug
    logger.info("Importing data into cryoweb staging area")
    logger.debug("Got Submission %s" % (submission))

    # If cryoweb has data, update submission message and return exception:
    # maybe another process is running or there is another type of problem
    if cryoweb_has_data():
        logger.error("Cryoweb has data!")

        # update submission status
        submission.status = ERROR
        submission.message = "Error in importing data: Cryoweb has data"
        submission.save()

        # send async message
        send_message(submission)

        raise CryoWebImportError("Cryoweb has data!")

    # this is the full path in docker container
    fullpath = submission.get_uploaded_file_path()

    # define command line
    cmd_line = "/usr/bin/psql -U {user} -h db {database}".format(
        database=database_name, user='cryoweb_insert_only')

    cmds = shlex.split(cmd_line)

    logger.debug("Executing: %s" % " ".join(cmds))

    try:
        result = subprocess.run(
            cmds,
            stdin=open(fullpath),
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            check=True,
            env={'PGPASSWORD': config('CRYOWEB_INSERT_ONLY_PW')},
            encoding='utf8'
            )

    except Exception as exc:
        # save a message in database
        submission.status = ERROR
        submission.message = "Error in importing data: %s" % (str(exc))
        submission.save()

        # send async message
        send_message(submission)

        # debug
        logger.error("error in calling upload_cryoweb: %s" % (exc))

        return False

    n_of_statements = len(result.stdout.split("\n"))
    logger.debug("%s statement executed" % n_of_statements)

    if len(result.stderr) > 0:
        for line in result.stderr.split("\n"):
            logger.error(line)

    logger.info("{filename} uploaded into {database}".format(
        filename=submission.uploaded_file.name, database=database_name))

    return True


# --- Upload data from cryoweb to UID


def fill_uid_breeds(submission):
    """Fill UID DictBreed model. Require a submission instance"""

    logger.info("fill_uid_breeds() started")

    # get submission language
    language = submission.gene_bank_country.label

    for v_breed_specie in VBreedsSpecies.objects.all():
        # get specie. Since I need a dictionary tables, DictSpecie is
        # already filled
        specie = DictSpecie.get_by_synonym(
            synonym=v_breed_specie.ext_species,
            language=language)

        # get country for breeds. Ideally will be the same of submission,
        # since the Italian cryoweb is supposed to contains italian breeds.
        # however, it could be possible to store data from other contries
        country, created = DictCountry.objects.get_or_create(
            label=v_breed_specie.efabis_country)

        # I could create a country from a v_breed_specie instance. That's
        # ok, maybe I could have a lot of breed from different countries and
        # a few organizations submitting them
        if created:
            logger.info("Created %s" % country)

        else:
            logger.debug("Found %s" % country)

        breed, created = DictBreed.objects.get_or_create(
            supplied_breed=v_breed_specie.efabis_mcname,
            specie=specie,
            country=country)

        if created:
            logger.info("Created %s:%s" % (breed, country))

        else:
            logger.debug("Found %s:%s" % (breed, country))

    logger.info("fill_uid_breeds() completed")


def fill_uid_names(submission):
    """Read VTransfer Views and fill name table"""

    # debug
    logger.info("called fill_uid_names()")

    # get all Vtransfer object
    for v_tranfer in VTransfer.objects.all():
        # no name manipulation. If two objects are indentical, there's no
        # duplicates.
        # HINT: The ramon example will be a issue in validation step
        name, created = Name.objects.get_or_create(
            name=v_tranfer.get_fullname(),
            submission=submission,
            owner=submission.owner)

        if created:
            logger.debug("Created %s" % name)

        else:
            logger.debug("Found %s" % name)

    logger.info("fill_uid_names() completed")


def fill_uid_animals(submission):
    """Helper function to fill animal data in UID animal table"""

    # debug
    logger.info("called fill_uid_animals()")

    # get submission language
    language = submission.gene_bank_country.label

    # get male and female DictSex objects from database
    male = DictSex.objects.get(label="male")
    female = DictSex.objects.get(label="female")

    # cycle over animals
    for v_animal in VAnimal.objects.all():
        # get specie translated by dictionary
        specie = DictSpecie.get_by_synonym(
            synonym=v_animal.ext_species,
            language=language)

        # get breed name and country through VBreedsSpecies model
        efabis_mcname = v_animal.efabis_mcname
        efabis_country = v_animal.efabis_country

        # get a country object
        country = DictCountry.objects.get(label=efabis_country)

        # a breed could be specie/country specific
        breed = DictBreed.objects.get(
            supplied_breed=efabis_mcname,
            specie=specie,
            country=country)

        logger.debug("Selected breed is %s" % (breed))

        # get name for this animal and for mother and father
        logger.debug("Getting %s as my name" % (v_animal.ext_animal))
        name = Name.objects.get(
            name=v_animal.ext_animal, submission=submission)

        logger.debug("Getting %s as father" % (v_animal.ext_sire))
        father = Name.objects.get(
            name=v_animal.ext_sire, submission=submission)

        logger.debug("Getting %s as mother" % (v_animal.ext_dam))
        mother = Name.objects.get(
            name=v_animal.ext_dam, submission=submission)

        # determine sex. Check for values
        if v_animal.ext_sex == 'm':
            sex = male

        elif v_animal.ext_sex == 'f':
            sex = female

        else:
            raise CryoWebImportError(
                "Unknown sex '%s' for '%s'" % (v_animal.ext_sex, v_animal))

        # checking accuracy
        accuracy = MISSING

        if v_animal.latitude and v_animal.longitude:
            accuracy = UNKNOWN

        # create a new object. Using defaults to avoid collisions when
        # updating data
        defaults = {
            'alternative_id': v_animal.db_animal,
            'breed': breed,
            'sex': sex,
            'father': father,
            'mother': mother,
            'birth_date': v_animal.birth_dt,
            'birth_location_latitude': v_animal.latitude,
            'birth_location_longitude': v_animal.longitude,
            'birth_location_accuracy': accuracy,
            'description': v_animal.comment,
            'owner': submission.owner
        }

        animal, created = Animal.objects.update_or_create(
            name=name,
            defaults=defaults)

        if created:
            logger.debug("Created %s" % animal)

        else:
            logger.debug("Updating %s" % animal)

    # create a validation summary object and set all_count
    validation_summary, created = ValidationSummary.objects.get_or_create(
        submission=submission, type="animal")

    if created:
        logger.debug(
            "ValidationSummary animal created for submission %s" % submission)

    # reset counts
    validation_summary.reset_all_count()

    # debug
    logger.info("fill_uid_animals() completed")


def fill_uid_samples(submission):
    """Helper function to fill animal data in UID animal table"""

    # debug
    logger.info("called fill_uid_samples()")

    for v_vessel in VVessels.objects.all():
        # get name for this sample. Need to insert it
        name, created = Name.objects.get_or_create(
            name=v_vessel.ext_vessel,
            submission=submission,
            owner=submission.owner)

        if created:
            logger.debug("Created %s" % name)

        else:
            logger.debug("Found %s" % name)

        # get animal object using name
        animal = Animal.objects.get(
            name__name=v_vessel.ext_animal,
            name__submission=submission)

        # get a organism part. Organism parts need to be in lowercases
        organism_part, created = DictUberon.objects.get_or_create(
            label=v_vessel.get_organism_part().lower()
        )

        if created:
            logger.info("Created %s" % organism_part)

        else:
            logger.debug("Found %s" % organism_part)

        # get a v_animal instance to get access to animal birth date
        v_animal = VAnimal.objects.get(db_animal=v_vessel.db_animal)

        # derive animal age at collection. THis function deals with NULL valies
        animal_age_at_collection, time_units = image_timedelta(
            v_vessel.production_dt, v_animal.birth_dt)

        # create a new object. Using defaults to avoid collisions when
        # updating data
        defaults = {
            'alternative_id': v_vessel.db_vessel,
            'collection_date': v_vessel.production_dt,
            'protocol': v_vessel.get_protocol_name(),
            'organism_part': organism_part,
            'animal': animal,
            'description': v_vessel.comment,
            'owner': submission.owner,
            'animal_age_at_collection': animal_age_at_collection,
            'animal_age_at_collection_units': time_units
        }

        sample, created = Sample.objects.update_or_create(
            name=name,
            defaults=defaults)

        if created:
            logger.debug("Created %s" % sample)

        else:
            logger.debug("Updating %s" % sample)

    # create a validation summary object and set all_count
    validation_summary, created = ValidationSummary.objects.get_or_create(
        submission=submission, type="sample")

    if created:
        logger.debug(
            "ValidationSummary animal created for submission %s" % submission)

    # reset counts
    validation_summary.reset_all_count()

    # debug
    logger.info("fill_uid_samples() completed")


def cryoweb_import(submission):
    """Import data from cryoweb stage database into UID

    :submission: a submission instance
    """

    # debug
    logger.info("Importing from cryoweb staging area")

    try:
        # check UID status. get an exception if database is not initialized
        check_UID(submission)

        # BREEDS
        fill_uid_breeds(submission)

        # NAME
        fill_uid_names(submission)

        # ANIMALS
        fill_uid_animals(submission)

        # SAMPLES
        fill_uid_samples(submission)

    except Exception as exc:
        # save a message in database
        submission.status = ERROR
        submission.message = "Error in importing data: %s" % (str(exc))
        submission.save()

        # send async message
        send_message(submission)

        # debug
        logger.error("error in importing from cryoweb: %s" % (exc))
        logger.exception(exc)

        return False

    else:
        message = "Cryoweb import completed for submission: %s" % (
            submission.id)

        submission.message = message
        submission.status = LOADED
        submission.save()

        # send async message
        send_message(submission, send_validation=True)

    logger.info("Import from staging area is complete")

    return True


1		#!/usr/bin/env python3
2		# -- coding: utf-8 --
3		"""
4		Created on Mon May 14 10:28:39 2018
5
6		@author: Paolo Cozzi <[email protected]>
7		"""
8
9		# --- import
10
11		import logging
12		import os
13		import shlex
14		import subprocess
15		import asyncio
16
17		from decouple import AutoConfig
18
19		from django.conf import settings
20
21		from common.constants import LOADED, ERROR, MISSING, UNKNOWN, STATUSES
22		from common.helpers import image_timedelta, send_message_to_websocket
23		from image_app.models import (
24		Animal, DictBreed, DictCountry, DictSex, DictSpecie, Name, Sample,
25		Submission, DictUberon)
26		from language.helpers import check_species_synonyms
27		from validation.helpers import construct_validation_message
28		from validation.models import ValidationSummary
29
30		from .models import db_has_data as cryoweb_has_data
31		from .models import VAnimal, VBreedsSpecies, VTransfer, VVessels
32
33		# Get an instance of a logger
34		logger = logging.getLogger(__name__)
35
36
37		# --- check functions
38
39
40		# a function to detect if cryoweb species have synonyms or not
41		def check_species(country):
42		"""Check all cryoweb species for a synonym in a supplied language or
43		the default one, ie: check_species(country). country is an
44		image_app.models.DictCountry.label"""
45
46		# get all species using view
47		words = VBreedsSpecies.get_all_species()
48
49		# for logging purposes
50		database_name = settings.DATABASES['cryoweb']['NAME']
51
52		if len(words) == 0:
53		raise CryoWebImportError(
54		"You have no species in %s database" % database_name)
55
56		# debug
57		logger.debug("Got %s species from %s" % (words, database_name))
58
59		# check if every word as a synonym (a specie)
60		# (And create synonyms if don't exist)
61		return check_species_synonyms(words, country, create=True)
62
63
64		# a function specific for cryoweb import path to ensure that all required
65		# fields in UID are present. There could be a function like this in others
66		# import paths
67		def check_UID(submission):
68		"""A function to ensure that UID is valid before data upload. Specific
69		to the module where is called from"""
70
71		logger.debug("Checking UID")
72
73		# check that dict sex table contains data
74		if len(DictSex.objects.all()) == 0:
75		raise CryoWebImportError("You have to upload DictSex data")
76
77		# test for specie synonyms in submission language or defaul one
78		# otherwise, fill synonym table with new terms then throw exception
79		if not check_species(submission.gene_bank_country):
80		raise CryoWebImportError("Some species haven't a synonym!")
81
82		# return a status
83		return True
84
85
86		# A class to deal with cryoweb import errors
87		class CryoWebImportError(Exception):
88		pass
89
90
91	View Code Duplication	def send_message(submission_obj, send_validation=False):
		0 ignored issues – show Duplication introduced 2019-06-20 15:48 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
92		"""
93		Update submission.status and submission message using django
94		channels
95
96		Args:
97		submission_obj (image_app.models.Submission): an UID submission
98		object
99		send_validation (bool): send validation message or not
100		"""
101
102		# define a message to send
103		message = {
104		'message': STATUSES.get_value_display(submission_obj.status),
105		'notification_message': submission_obj.message,
106		}
107
108		# if validation message is needed, add to the final message
109		if send_validation:
110		message['validation_message'] = construct_validation_message(
111		submission_obj)
112
113		# now send the message to its submission
114		asyncio.get_event_loop().run_until_complete(
115		send_message_to_websocket(
116		message,
117		submission_obj.pk
118		)
119		)
120
121
122		# --- Upload data into cryoweb database
123		def upload_cryoweb(submission_id):
124		"""Imports backup into the cryoweb db
125
126		This function uses the container's installation of psql to import a backup
127		file into the "cryoweb" database. The imported backup file is
128		the last inserted into the image's table image_app_submission.
129
130		:submission_id: the submission primary key
131		"""
132
133		# define some useful variables
134		database_name = settings.DATABASES['cryoweb']['NAME']
135
136		# define a decouple config object
137		config_dir = os.path.join(settings.BASE_DIR, 'image')
138		config = AutoConfig(search_path=config_dir)
139
140		# get a submission object
141		submission = Submission.objects.get(pk=submission_id)
142
143		# debug
144		logger.info("Importing data into cryoweb staging area")
145		logger.debug("Got Submission %s" % (submission))
146
147		# If cryoweb has data, update submission message and return exception:
148		# maybe another process is running or there is another type of problem
149		if cryoweb_has_data():
150		logger.error("Cryoweb has data!")
151
152		# update submission status
153		submission.status = ERROR
154		submission.message = "Error in importing data: Cryoweb has data"
155		submission.save()
156
157		# send async message
158		send_message(submission)
159
160		raise CryoWebImportError("Cryoweb has data!")
161
162		# this is the full path in docker container
163		fullpath = submission.get_uploaded_file_path()
164
165		# define command line
166		cmd_line = "/usr/bin/psql -U {user} -h db {database}".format(
167		database=database_name, user='cryoweb_insert_only')
168
169		cmds = shlex.split(cmd_line)
170
171		logger.debug("Executing: %s" % " ".join(cmds))
172
173		try:
174		result = subprocess.run(
175		cmds,
176		stdin=open(fullpath),
177		stdout=subprocess.PIPE,
178		stderr=subprocess.PIPE,
179		check=True,
180		env={'PGPASSWORD': config('CRYOWEB_INSERT_ONLY_PW')},
181		encoding='utf8'
182		)
183
184		except Exception as exc:
185		# save a message in database
186		submission.status = ERROR
187		submission.message = "Error in importing data: %s" % (str(exc))
188		submission.save()
189
190		# send async message
191		send_message(submission)
192
193		# debug
194		logger.error("error in calling upload_cryoweb: %s" % (exc))
195
196		return False
197
198		n_of_statements = len(result.stdout.split("\n"))
199		logger.debug("%s statement executed" % n_of_statements)
200
201		if len(result.stderr) > 0:
202		for line in result.stderr.split("\n"):
203		logger.error(line)
204
205		logger.info("{filename} uploaded into {database}".format(
206		filename=submission.uploaded_file.name, database=database_name))
207
208		return True
209
210
211		# --- Upload data from cryoweb to UID
212
213
214		def fill_uid_breeds(submission):
215		"""Fill UID DictBreed model. Require a submission instance"""
216
217		logger.info("fill_uid_breeds() started")
218
219		# get submission language
220		language = submission.gene_bank_country.label
221
222		for v_breed_specie in VBreedsSpecies.objects.all():
223		# get specie. Since I need a dictionary tables, DictSpecie is
224		# already filled
225		specie = DictSpecie.get_by_synonym(
226		synonym=v_breed_specie.ext_species,
227		language=language)
228
229		# get country for breeds. Ideally will be the same of submission,
230		# since the Italian cryoweb is supposed to contains italian breeds.
231		# however, it could be possible to store data from other contries
232		country, created = DictCountry.objects.get_or_create(
233		label=v_breed_specie.efabis_country)
234
235		# I could create a country from a v_breed_specie instance. That's
236		# ok, maybe I could have a lot of breed from different countries and
237		# a few organizations submitting them
238		if created:
239		logger.info("Created %s" % country)
240
241		else:
242		logger.debug("Found %s" % country)
243
244		breed, created = DictBreed.objects.get_or_create(
245		supplied_breed=v_breed_specie.efabis_mcname,
246		specie=specie,
247		country=country)
248
249		if created:
250		logger.info("Created %s:%s" % (breed, country))
251
252		else:
253		logger.debug("Found %s:%s" % (breed, country))
254
255		logger.info("fill_uid_breeds() completed")
256
257
258		def fill_uid_names(submission):
259		"""Read VTransfer Views and fill name table"""
260
261		# debug
262		logger.info("called fill_uid_names()")
263
264		# get all Vtransfer object
265		for v_tranfer in VTransfer.objects.all():
266		# no name manipulation. If two objects are indentical, there's no
267		# duplicates.
268		# HINT: The ramon example will be a issue in validation step
269		name, created = Name.objects.get_or_create(
270		name=v_tranfer.get_fullname(),
271		submission=submission,
272		owner=submission.owner)
273
274		if created:
275		logger.debug("Created %s" % name)
276
277		else:
278		logger.debug("Found %s" % name)
279
280		logger.info("fill_uid_names() completed")
281
282
283		def fill_uid_animals(submission):
284		"""Helper function to fill animal data in UID animal table"""
285
286		# debug
287		logger.info("called fill_uid_animals()")
288
289		# get submission language
290		language = submission.gene_bank_country.label
291
292		# get male and female DictSex objects from database
293		male = DictSex.objects.get(label="male")
294		female = DictSex.objects.get(label="female")
295
296		# cycle over animals
297		for v_animal in VAnimal.objects.all():
298		# get specie translated by dictionary
299		specie = DictSpecie.get_by_synonym(
300		synonym=v_animal.ext_species,
301		language=language)
302
303		# get breed name and country through VBreedsSpecies model
304		efabis_mcname = v_animal.efabis_mcname
305		efabis_country = v_animal.efabis_country
306
307		# get a country object
308		country = DictCountry.objects.get(label=efabis_country)
309
310		# a breed could be specie/country specific
311		breed = DictBreed.objects.get(
312		supplied_breed=efabis_mcname,
313		specie=specie,
314		country=country)
315
316		logger.debug("Selected breed is %s" % (breed))
317
318		# get name for this animal and for mother and father
319		logger.debug("Getting %s as my name" % (v_animal.ext_animal))
320		name = Name.objects.get(
321		name=v_animal.ext_animal, submission=submission)
322
323		logger.debug("Getting %s as father" % (v_animal.ext_sire))
324		father = Name.objects.get(
325		name=v_animal.ext_sire, submission=submission)
326
327		logger.debug("Getting %s as mother" % (v_animal.ext_dam))
328		mother = Name.objects.get(
329		name=v_animal.ext_dam, submission=submission)
330
331		# determine sex. Check for values
332		if v_animal.ext_sex == 'm':
333		sex = male
334
335		elif v_animal.ext_sex == 'f':
336		sex = female
337
338		else:
339		raise CryoWebImportError(
340		"Unknown sex '%s' for '%s'" % (v_animal.ext_sex, v_animal))
341
342		# checking accuracy
343		accuracy = MISSING
344
345		if v_animal.latitude and v_animal.longitude:
346		accuracy = UNKNOWN
347
348		# create a new object. Using defaults to avoid collisions when
349		# updating data
350		defaults = {
351		'alternative_id': v_animal.db_animal,
352		'breed': breed,
353		'sex': sex,
354		'father': father,
355		'mother': mother,
356		'birth_date': v_animal.birth_dt,
357		'birth_location_latitude': v_animal.latitude,
358		'birth_location_longitude': v_animal.longitude,
359		'birth_location_accuracy': accuracy,
360		'description': v_animal.comment,
361		'owner': submission.owner
362		}
363
364		animal, created = Animal.objects.update_or_create(
365		name=name,
366		defaults=defaults)
367
368		if created:
369		logger.debug("Created %s" % animal)
370
371		else:
372		logger.debug("Updating %s" % animal)
373
374		# create a validation summary object and set all_count
375		validation_summary, created = ValidationSummary.objects.get_or_create(
376		submission=submission, type="animal")
377
378		if created:
379		logger.debug(
380		"ValidationSummary animal created for submission %s" % submission)
381
382		# reset counts
383		validation_summary.reset_all_count()
384
385		# debug
386		logger.info("fill_uid_animals() completed")
387
388
389		def fill_uid_samples(submission):
390		"""Helper function to fill animal data in UID animal table"""
391
392		# debug
393		logger.info("called fill_uid_samples()")
394
395		for v_vessel in VVessels.objects.all():
396		# get name for this sample. Need to insert it
397		name, created = Name.objects.get_or_create(
398		name=v_vessel.ext_vessel,
399		submission=submission,
400		owner=submission.owner)
401
402		if created:
403		logger.debug("Created %s" % name)
404
405		else:
406		logger.debug("Found %s" % name)
407
408		# get animal object using name
409		animal = Animal.objects.get(
410		name__name=v_vessel.ext_animal,
411		name__submission=submission)
412
413		# get a organism part. Organism parts need to be in lowercases
414		organism_part, created = DictUberon.objects.get_or_create(
415		label=v_vessel.get_organism_part().lower()
416		)
417
418		if created:
419		logger.info("Created %s" % organism_part)
420
421		else:
422		logger.debug("Found %s" % organism_part)
423
424		# get a v_animal instance to get access to animal birth date
425		v_animal = VAnimal.objects.get(db_animal=v_vessel.db_animal)
426
427		# derive animal age at collection. THis function deals with NULL valies
428		animal_age_at_collection, time_units = image_timedelta(
429		v_vessel.production_dt, v_animal.birth_dt)
430
431		# create a new object. Using defaults to avoid collisions when
432		# updating data
433		defaults = {
434		'alternative_id': v_vessel.db_vessel,
435		'collection_date': v_vessel.production_dt,
436		'protocol': v_vessel.get_protocol_name(),
437		'organism_part': organism_part,
438		'animal': animal,
439		'description': v_vessel.comment,
440		'owner': submission.owner,
441		'animal_age_at_collection': animal_age_at_collection,
442		'animal_age_at_collection_units': time_units
443		}
444
445		sample, created = Sample.objects.update_or_create(
446		name=name,
447		defaults=defaults)
448
449		if created:
450		logger.debug("Created %s" % sample)
451
452		else:
453		logger.debug("Updating %s" % sample)
454
455		# create a validation summary object and set all_count
456		validation_summary, created = ValidationSummary.objects.get_or_create(
457		submission=submission, type="sample")
458
459		if created:
460		logger.debug(
461		"ValidationSummary animal created for submission %s" % submission)
462
463		# reset counts
464		validation_summary.reset_all_count()
465
466		# debug
467		logger.info("fill_uid_samples() completed")
468
469
470		def cryoweb_import(submission):
471		"""Import data from cryoweb stage database into UID
472
473		:submission: a submission instance
474		"""
475
476		# debug
477		logger.info("Importing from cryoweb staging area")
478
479		try:
480		# check UID status. get an exception if database is not initialized
481		check_UID(submission)
482
483		# BREEDS
484		fill_uid_breeds(submission)
485
486		# NAME
487		fill_uid_names(submission)
488
489		# ANIMALS
490		fill_uid_animals(submission)
491
492		# SAMPLES
493		fill_uid_samples(submission)
494
495		except Exception as exc:
496		# save a message in database
497		submission.status = ERROR
498		submission.message = "Error in importing data: %s" % (str(exc))
499		submission.save()
500
501		# send async message
502		send_message(submission)
503
504		# debug
505		logger.error("error in importing from cryoweb: %s" % (exc))
506		logger.exception(exc)
507
508		return False
509
510		else:
511		message = "Cryoweb import completed for submission: %s" % (
512		submission.id)
513
514		submission.message = message
515		submission.status = LOADED
516		submission.save()
517
518		# send async message
519		send_message(submission, send_validation=True)
520
521		logger.info("Import from staging area is complete")
522
523		return True
524

cnr-ibba / IMAGE-InjectTool

Pull Request — master (#35)

cryoweb.helpers.fill_uid_breeds() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like