Passed
Pull Request — master (#26)
by Paolo
02:00
created

image_app.models.Animal.get_father_relationship()   A

Complexity

Conditions 3

Size

Total Lines 14
Code Lines 6

Duplication

Lines 0
Ratio 0 %

Importance

Changes 0
Metric Value
eloc 6
dl 0
loc 14
rs 10
c 0
b 0
f 0
cc 3
nop 1
1
2
import logging
3
import os
4
import shlex
5
6
from django.contrib.auth.models import User
7
from django.db import connections, models
8
from django.db.models import Func, Value, F
9
from django.db.models.signals import post_save
10
from django.dispatch import receiver
11
from django.urls import reverse
12
from django.utils import timezone
13
14
from common.fields import ProtectedFileField
15
from common.constants import (
16
    OBO_URL, STATUSES, CONFIDENCES, NAME_STATUSES, ACCURACIES, WAITING, LOADED,
17
    MISSING, DATA_TYPES, TIME_UNITS)
18
19
from .helpers import format_attribute
20
21
# Get an instance of a logger
22
logger = logging.getLogger(__name__)
23
24
25
class Replace(Func):
26
    function = 'REPLACE'
27
28
29
# --- Mixins
30
31
32
# Adding a classmethod to Category if you want to enable truncate
33
# https://books.agiliq.com/projects/django-orm-cookbook/en/latest/truncate.html
34
class BaseMixin(object):
35
    """Base class for UID tables. It implement common stuff for all UID
36
    tables::
37
38
        from image_app.models import BaseMixin
39
40
        class Submission(BaseMixin):
41
            pass
42
43
    """
44
45
    @classmethod
46
    def truncate(cls):
47
        """
48
        Truncate table data and restart indexes from 0::
49
50
            from image_app.models import Submission
51
52
            Submission.truncate()
53
        """
54
55
        # Django.db.connections is a dictionary-like object that allows you
56
        # to retrieve a specific connection using its alias
57
        with connections["default"].cursor() as cursor:
58
            statement = "TRUNCATE TABLE {0} RESTART IDENTITY CASCADE".format(
59
                cls._meta.db_table)
60
            logger.debug(statement)
61
            cursor.execute(statement)
62
63
64
class BioSampleMixin(BaseMixin):
65
    """
66
    Common methods for animal and samples useful in biosample generation
67
    Need to called with data into biosample or animals::
68
69
        from image_app.models import Animal
70
71
        animal = Animal.objects.get(pk=1)
72
        biosample_data = animal.to_biosample()
73
74
    """
75
76
    def __str__(self):
77
        return str(self.name)
78
79
    @property
80
    def person(self):
81
        """Retrieve :py:class:`Person` information from owner relationship"""
82
83
        return self.owner.person
84
85
    @property
86
    def organization(self):
87
        """Return :py:class:`Organization` relationship from related
88
        :py:class:`Submission` object"""
89
90
        return self.name.submission.organization
91
92
    @property
93
    def gene_bank_country(self):
94
        """Return :py:class:`DictCountry` relationship from related
95
        :py:class:`Submission` object"""
96
97
        return self.name.submission.gene_bank_country
98
99
    @property
100
    def gene_bank_name(self):
101
        """Return gene bank name from related :py:class:`Submission` object"""
102
103
        return self.name.submission.gene_bank_name
104
105
    @property
106
    def data_source_id(self):
107
        """Get Data source id (original animal/sample name) from related
108
        :py:class:`Name` object"""
109
110
        return self.name.name
111
112
    @property
113
    def submission(self):
114
        """Get related :py:class:`Submission` object throgh :py:class:`Name`
115
        """
116
        return self.name.submission
117
118
    @property
119
    def specie(self):
120
        raise NotImplementedError(
121
            "You need to define this method in your class")
122
123
    @property
124
    def biosample_id(self):
125
        """Get the biosample_id of an object (need to be submitted and
126
        retrieved sing USI)"""
127
128
        return self.name.biosample_id
129
130
    def get_attributes(self):
131
        """Common attribute definition required from Animal and samples. Need
132
        to be called inside Animal/sample get_atribute method. Keys
133
        is the name in metadata rules
134
135
        Returns:
136
            dict: a dictionary object
137
        """
138
139
        attributes = {}
140
141
        attributes['Data source ID'] = format_attribute(
142
            value=self.data_source_id)
143
144
        attributes['Alternative id'] = format_attribute(
145
            value=self.alternative_id)
146
147
        # HINT: this is a mandatory biosample field: could be removed from
148
        # attributes?
149
        attributes['Description'] = format_attribute(
150
            value=self.description)
151
152
        attributes["Project"] = format_attribute(
153
            value="IMAGE")
154
155
        attributes['Submission title'] = format_attribute(
156
            value=self.submission.title)
157
158
        attributes['Submission description'] = format_attribute(
159
            value=self.submission.description)
160
161
        attributes['Person last name'] = format_attribute(
162
            value=self.owner.last_name)
163
164
        attributes['Person initial'] = format_attribute(
165
            value=self.person.initials)
166
167
        attributes['Person first name'] = format_attribute(
168
            value=self.owner.first_name)
169
170
        attributes['Person email'] = format_attribute(
171
            value="mailto:%s" % (self.owner.email))
172
173
        attributes['Person affiliation'] = format_attribute(
174
            value=self.person.affiliation.name)
175
176
        attributes['Person role'] = self.person.role.format_attribute()
177
178
        attributes['Organization name'] = format_attribute(
179
            value=self.organization.name)
180
181
        attributes['Organization address'] = format_attribute(
182
            value=self.organization.address)
183
184
        attributes['Organization uri'] = format_attribute(
185
            value=self.organization.URI)
186
187
        attributes['Organization country'] = \
188
            self.organization.country.format_attribute()
189
190
        attributes[
191
            'Organization role'] = self.organization.role.format_attribute()
192
193
        # this could be present or not
194
        if self.name.publication:
195
            attributes['Publication DOI'] = format_attribute(
196
                value=self.name.publication.doi)
197
198
        attributes['Gene bank name'] = format_attribute(
199
            value=self.gene_bank_name)
200
201
        attributes[
202
            'Gene bank country'] = self.gene_bank_country.format_attribute()
203
204
        attributes['Data source type'] = format_attribute(
205
            value=self.submission.get_datasource_type_display())
206
207
        attributes['Data source version'] = format_attribute(
208
            value=self.submission.datasource_version)
209
210
        attributes['Species'] = self.specie.format_attribute()
211
212
        return attributes
213
214
    def to_biosample(self, release_date=None):
215
        """
216
        Common stuff to generate a biosample object. Need to be called
217
        inside Animal/Sample to_biosample method
218
219
        Args:
220
            release_date (str): data will no be published before this day
221
                (YYYY-MM-DD)
222
223
        Returns:
224
            dict: a dictionary object
225
        """
226
227
        result = {}
228
229
        # define mandatory fields
230
        result['alias'] = self.biosample_alias
231
        result['title'] = self.name.name
232
233
        # in case of update, I need to provide the old accession in payload
234
        if self.biosample_id and self.biosample_id != '':
235
            result['accession'] = self.biosample_id
236
237
        if release_date:
238
            result['releaseDate'] = release_date
239
240
        else:
241
            now = timezone.now()
242
            result['releaseDate'] = str(now.date())
243
244
        result['taxonId'] = self.specie.taxon_id
245
246
        result['taxon'] = self.specie.label
247
248
        # define optinal fields
249
        if self.description:
250
            result['description'] = self.description
251
252
        # define attributes that will be customized in Animal and sample
253
        result['attributes'] = self.get_attributes()
254
255
        return result
256
257
    def __can_I(self, names):
258
        """
259
        Return True id self.status in statuses
260
261
        Args:
262
            names (list): a list of :py:class:`common.constants.STATUSES`
263
264
        Returns:
265
            bool
266
        """
267
268
        statuses = [x.value[0] for x in STATUSES if x.name in names]
269
270
        if self.submission.status not in statuses:
271
            return True
272
273
        else:
274
            return False
275
276
    def can_edit(self):
277
        """Returns True if I can edit a sample/animal according to submission
278
        status
279
280
        Returns:
281
            bool
282
        """
283
284
        names = ['waiting', 'submitted']
285
286
        return self.__can_I(names)
287
288
    def can_delete(self):
289
        """Returns True if I can delete a sample/animal according to submission
290
        status
291
292
        Returns:
293
            bool
294
        """
295
296
        names = ['waiting', 'submitted']
297
298
        return self.__can_I(names)
299
300
301
# --- Abstract classes
302
303
304
# helper classes
305
class DictBase(BaseMixin, models.Model):
306
    """
307
    Abstract class to be inherited to all dictionary tables. It models fields
308
    like ``label`` (the revised term like submitter or blood) and
309
    ``term`` (the ontology id as the final part of the URI link)
310
311
    The fixed part of the URI could be customized from :py:class:`Ontology`
312
    by setting ``library_name`` class attribute accordingly::
313
314
        class DictRole(DictBase):
315
            library_name = 'EFO'
316
    """
317
318
    library_name = None
319
320
    # if not defined, this table will have an own primary key
321
    label = models.CharField(
322
            max_length=255,
323
            blank=False,
324
            help_text="Example: submitter")
325
326
    term = models.CharField(
327
            max_length=255,
328
            blank=False,
329
            null=True,
330
            help_text="Example: EFO_0001741")
331
332
    class Meta:
333
        # Abstract base classes are useful when you want to put some common
334
        # information into a number of other models
335
        abstract = True
336
337
    def __str__(self):
338
        return "{label} ({term})".format(
339
                label=self.label,
340
                term=self.term)
341
342
    def format_attribute(self):
343
        """
344
        Format an object instance as a dictionary used by biosample, for
345
        example::
346
347
            [{
348
                'value': 'submitter',
349
                'terms': [{'url': 'http://www.ebi.ac.uk/efo/EFO_0001741'}]
350
            }]
351
352
        the fixed part of URI link is defined by ``library_name`` class
353
        attribute
354
        """
355
356
        if self.library_name is None:
357
            logger.warning("library_name not defined")
358
            library_uri = OBO_URL
359
360
        else:
361
            library = Ontology.objects.get(library_name=self.library_name)
362
            library_uri = library.library_uri
363
364
        return format_attribute(
365
            value=self.label,
366
            library_uri=library_uri,
367
            terms=self.term)
368
369
370
class Confidence(BaseMixin, models.Model):
371
    """
372
    Abstract class which add :ref:`confidence <Common confidences>`
373
    to models
374
    """
375
376
    # confidence field (enum)
377
    confidence = models.SmallIntegerField(
378
        choices=[x.value for x in CONFIDENCES],
379
        help_text='example: Manually Curated',
380
        null=True)
381
382
    class Meta:
383
        # Abstract base classes are useful when you want to put some common
384
        # information into a number of other models
385
        abstract = True
386
387
388
# --- dictionary tables
389
390
391
class DictRole(DictBase):
392
    """A class to model roles defined as childs of
393
    http://www.ebi.ac.uk/efo/EFO_0002012"""
394
395
    library_name = 'EFO'
396
397
    class Meta:
398
        # db_table will be <app_name>_<classname>
399
        verbose_name = "role"
400
        unique_together = (("label", "term"),)
401
402
403
class DictCountry(DictBase, Confidence):
404
    """A class to model contries defined by NCI Thesaurus OBO Edition
405
    https://www.ebi.ac.uk/ols/ontologies/ncit"""
406
407
    library_name = 'NCIT'
408
409
    class Meta:
410
        # db_table will be <app_name>_<classname>
411
        verbose_name = "country"
412
        verbose_name_plural = "countries"
413
        unique_together = (("label", "term"),)
414
        ordering = ['label']
415
416
417
class DictSex(DictBase):
418
    """A class to model sex as defined in PATO"""
419
420
    library_name = "PATO"
421
422
    class Meta:
423
        verbose_name = 'sex'
424
        verbose_name_plural = 'sex'
425
        unique_together = (("label", "term"),)
426
427
428
class DictUberon(DictBase, Confidence):
429
    """A class to model anatomies modeled in uberon"""
430
431
    library_name = "UBERON"
432
433
    class Meta:
434
        verbose_name = 'organism part'
435
        unique_together = (("label", "term"),)
436
437
438
class DictStage(DictBase, Confidence):
439
    """A class to developmental stages defined as descendants of
440
    descendants of EFO_0000399"""
441
442
    library_name = 'EFO'
443
444
    class Meta:
445
        # db_table will be <app_name>_<classname>
446
        verbose_name = "developmental stage"
447
        unique_together = (("label", "term"),)
448
449
450
class DictSpecie(DictBase, Confidence):
451
    """A class to model species defined by NCBI organismal classification
452
    http://www.ebi.ac.uk/ols/ontologies/ncbitaxon"""
453
454
    library_name = "NCBITaxon"
455
456
    @property
457
    def taxon_id(self):
458
        if not self.term or self.term == '':
459
            return None
460
461
        return int(self.term.split("_")[-1])
462
463
    class Meta:
464
        # db_table will be <app_name>_<classname>
465
        verbose_name = "specie"
466
        unique_together = (("label", "term"),)
467
468
    @classmethod
469
    def get_by_synonym(cls, synonym, language):
470
        """return an instance by synonym in supplied language or default one"""
471
472
        # get a queryset with speciesynonym
473
        qs = cls.objects.prefetch_related('speciesynonym_set')
474
475
        # annotate queryset by removing spaces from speciesynonym word
476
        qs = qs.annotate(
477
            new_word=Replace('speciesynonym__word', Value(" "), Value("")),
478
            language=F('speciesynonym__language__label'))
479
480
        # now remove spaces from synonym
481
        synonym = synonym.replace(" ", "")
482
483
        try:
484
            specie = qs.get(
485
                new_word=synonym,
486
                language=language)
487
488
        except cls.DoesNotExist:
489
            specie = qs.get(
490
                new_word=synonym,
491
                language="United Kingdom")
492
493
        return specie
494
495
496
class DictBreed(Confidence):
497
    """A class to deal with breed objects and their ontologies"""
498
499
    library_name = "LBO"
500
501
    # this was the description field in cryoweb v_breeds_species tables
502
    supplied_breed = models.CharField(max_length=255, blank=False)
503
    mapped_breed = models.CharField(max_length=255, blank=False, null=True)
504
505
    mapped_breed_term = models.CharField(
506
            max_length=255,
507
            blank=False,
508
            null=True,
509
            help_text="Example: LBO_0000347")
510
511
    # using a constraint for country.
512
    country = models.ForeignKey(
513
        'DictCountry',
514
        on_delete=models.PROTECT)
515
516
    # using a constraint for specie
517
    specie = models.ForeignKey(
518
        'DictSpecie',
519
        on_delete=models.PROTECT)
520
521
    class Meta:
522
        verbose_name = 'Breed'
523
        unique_together = (("supplied_breed", "specie", "country"),)
524
525
    def __str__(self):
526
        return "{supplied} ({mapped}, {specie})".format(
527
            supplied=self.supplied_breed,
528
            mapped=self.mapped_breed,
529
            specie=self.specie.label)
530
531
    def format_attribute(self):
532
        """Format mapped_breed attribute (with its ontology). Return None if
533
        no mapped_breed"""
534
535
        if not self.mapped_breed or not self.mapped_breed_term:
536
            return None
537
538
        library = Ontology.objects.get(library_name=self.library_name)
539
        library_uri = library.library_uri
540
541
        return format_attribute(
542
            value=self.mapped_breed,
543
            library_uri=library_uri,
544
            terms=self.mapped_breed_term)
545
546
547
# --- Other tables tables
548
549
550
class Name(BaseMixin, models.Model):
551
    """Model UID names: define a name (sample or animal) unique for each
552
    data submission"""
553
554
    # two different animal may have the same name. Its unicity depens on
555
    # data source name and version
556
    name = models.CharField(
557
            max_length=255,
558
            blank=False,
559
            null=False)
560
561
    submission = models.ForeignKey(
562
        'Submission',
563
        db_index=True,
564
        related_name='name_set',
565
        on_delete=models.CASCADE)
566
567
    # This need to be assigned after submission
568
    # HINT: this column should be UNIQUE?
569
    biosample_id = models.CharField(max_length=255, blank=True, null=True)
570
571
    # '+' instructs Django that we don’t need this reverse relationship
572
    owner = models.ForeignKey(
573
        User,
574
        related_name='+',
575
        on_delete=models.CASCADE)
576
577
    # a column to track submission status
578
    status = models.SmallIntegerField(
579
            choices=[x.value for x in STATUSES if x.name in NAME_STATUSES],
580
            help_text='example: Submitted',
581
            default=LOADED)
582
583
    last_changed = models.DateTimeField(
584
        auto_now_add=True,
585
        blank=True,
586
        null=True)
587
588
    last_submitted = models.DateTimeField(
589
        blank=True,
590
        null=True)
591
592
    publication = models.ForeignKey(
593
        'Publication',
594
        null=True,
595
        blank=True,
596
        related_name='name_set',
597
        on_delete=models.SET_NULL)
598
599
    class Meta:
600
        unique_together = (("name", "submission"),)
601
602
    def __str__(self):
603
        # HINT: should I return biosampleid if defined?
604
        return "%s (Submission: %s)" % (self.name, self.submission_id)
605
606
607
class Animal(BioSampleMixin, models.Model):
608
    """
609
    Class to model Animal (Organism). Inherits from :py:class:`BioSampleMixin`,
610
    related to :py:class:`Name` through ``OneToOne`` relationship to model
611
    Animal name (Data source id), and with the same relationship to model
612
    ``mother`` and ``father`` of such animal. In case that parents are unknown,
613
    could be linked with Unkwnon animals for cryoweb data or doens't have
614
    relationship.Linked to :py:class:`DictBreed` dictionary
615
    table to model info on species and breed. Linked to
616
    :py:class:`Sample` to model Samples (Specimen from organims)::
617
618
        from image_app.models import Animal
619
620
        # get animal using primary key
621
        animal = Animal.objects.get(pk=1)
622
623
        # get animal name
624
        data_source_id = animal.name.name
625
626
        # get animal's parents
627
        mother = animal.mother
628
        father = animal.father
629
630
        # get breed and species info
631
        print(animal.breed.supplied_breed)
632
        print(animal.breed.specie.label)
633
634
        # get all samples (specimen) for this animals
635
        samples = animal.sample_set.all()
636
    """
637
638
    # an animal name has a entry in name table
639
    name = models.OneToOneField(
640
        'Name',
641
        on_delete=models.CASCADE)
642
643
    # alternative id will store the internal id in data source
644
    alternative_id = models.CharField(max_length=255, blank=True, null=True)
645
646
    description = models.CharField(max_length=255, blank=True, null=True)
647
648
    # HINT: link to a term list table?
649
    material = models.CharField(
650
        max_length=255,
651
        default="Organism",
652
        editable=False)
653
654
    breed = models.ForeignKey(
655
        'DictBreed',
656
        db_index=True,
657
        on_delete=models.PROTECT)
658
659
    # species is in DictBreed table
660
661
    # using a constraint for sex
662
    sex = models.ForeignKey(
663
        'DictSex',
664
        null=True,
665
        on_delete=models.PROTECT)
666
667
    # check that father and mother are defined using Foreign Keys
668
    # HINT: mother and father are not mandatory in all datasource
669
    father = models.ForeignKey(
670
        'Name',
671
        on_delete=models.CASCADE,
672
        null=True,
673
        related_name='father_of')
674
675
    mother = models.ForeignKey(
676
        'Name',
677
        on_delete=models.CASCADE,
678
        null=True,
679
        related_name='mother_of')
680
681
    birth_date = models.DateField(
682
        blank=True,
683
        null=True,
684
        help_text='example: 2019-04-01')
685
686
    # TODO: need to set this value? How?
687
    birth_location = models.CharField(
688
        max_length=255,
689
        blank=True,
690
        null=True)
691
692
    birth_location_latitude = models.FloatField(blank=True, null=True)
693
    birth_location_longitude = models.FloatField(blank=True, null=True)
694
695
    # accuracy field (enum)
696
    birth_location_accuracy = models.SmallIntegerField(
697
        choices=[x.value for x in ACCURACIES],
698
        help_text='example: unknown accuracy level, country level',
699
        null=False,
700
        blank=False,
701
        default=MISSING)
702
703
    owner = models.ForeignKey(
704
        User,
705
        related_name='animals',
706
        on_delete=models.CASCADE)
707
708
    @property
709
    def specie(self):
710
        return self.breed.specie
711
712
    @property
713
    def biosample_alias(self):
714
        return 'IMAGEA{0:09d}'.format(self.id)
715
716
    def get_attributes(self):
717
        """Return attributes like biosample needs"""
718
719
        attributes = super().get_attributes()
720
721
        attributes["Material"] = format_attribute(
722
            value="organism", terms="OBI_0100026")
723
724
        # TODO: how to model derived from (mother/father)?
725
726
        attributes['Supplied breed'] = format_attribute(
727
            value=self.breed.supplied_breed)
728
729
        # HINT: Ideally, I could retrieve an ontology id for countries
730
        attributes['EFABIS Breed country'] = format_attribute(
731
            value=self.breed.country.label)
732
733
        attributes['Mapped breed'] = self.breed.format_attribute()
734
735
        attributes['Sex'] = self.sex.format_attribute()
736
737
        # a datetime object should be not be converted in string here,
738
        # otherwise will not be filtered if NULL
739
        attributes['Birth date'] = format_attribute(
740
            value=self.birth_date, units="YYYY-MM-DD")
741
742
        attributes["Birth location"] = format_attribute(
743
            value=self.birth_location)
744
745
        attributes["Birth location longitude"] = format_attribute(
746
            value=self.birth_location_longitude,
747
            units="decimal degrees")
748
749
        attributes["Birth location latitude"] = format_attribute(
750
            value=self.birth_location_latitude,
751
            units="decimal degrees")
752
753
        attributes["Birth location accuracy"] = format_attribute(
754
            value=self.get_birth_location_accuracy_display())
755
756
        # filter out empty values
757
        attributes = {k: v for k, v in attributes.items() if v is not None}
758
759
        return attributes
760
761
    def get_relationship(self):
762
        """Get a relationship to this animal (call this method from a related
763
        object to get a connection to this element)"""
764
765
        # if animal is already uploaded I will use accession as
766
        # relationship key. This biosample id could be tested in validation
767
        if self.biosample_id and self.biosample_id != '':
768
            return {
769
                "accession": self.biosample_id,
770
                "relationshipNature": "derived from",
771
            }
772
        else:
773
            return {
774
                "alias": self.biosample_alias,
775
                "relationshipNature": "derived from",
776
            }
777
778
    def get_father_relationship(self):
779
        """Get a relationship with father if possible"""
780
781
        # get father of this animal
782
        if self.father is None:
783
            return None
784
785
        # cryoweb could have unkwown animals. They are Names without
786
        # relationship with Animal table
787
        if not hasattr(self.father, "animal"):
788
            return None
789
790
        else:
791
            return self.father.animal.get_relationship()
792
793
    def get_mother_relationship(self):
794
        """Get a relationship with mother if possible"""
795
796
        # get mother of this animal
797
        if self.mother is None:
798
            return None
799
800
        # cryoweb could have unkwown animals. They are Names without
801
        # relationship with Animal table
802
        if not hasattr(self.mother, "animal"):
803
            return None
804
805
        else:
806
            return self.mother.animal.get_relationship()
807
808
    def to_biosample(self, release_date=None):
809
        """get a json from animal for biosample submission"""
810
811
        # call methods defined in BioSampleMixin and get result
812
        # with USI mandatory keys and attributes
813
        result = super().to_biosample(release_date)
814
815
        # define relationship with mother and father (if possible)
816
        result['sampleRelationships'] = []
817
818
        father_relationship = self.get_father_relationship()
819
820
        if father_relationship is not None:
821
            result['sampleRelationships'].append(father_relationship)
822
823
        mother_relationship = self.get_mother_relationship()
824
825
        if mother_relationship is not None:
826
            result['sampleRelationships'].append(mother_relationship)
827
828
        return result
829
830
    def get_absolute_url(self):
831
        return reverse("animals:detail", kwargs={"pk": self.pk})
832
833
834
class Sample(BioSampleMixin, models.Model):
835
    # a sample name has a entry in name table
836
    # this is a One2One foreign key
837
    name = models.OneToOneField(
838
        'Name',
839
        on_delete=models.CASCADE)
840
841
    # db_vessel in data source
842
    alternative_id = models.CharField(max_length=255, blank=True, null=True)
843
844
    description = models.CharField(max_length=255, blank=True, null=True)
845
846
    # HINT: link to a term list table?
847
    material = models.CharField(
848
        max_length=255,
849
        default="Specimen from Organism",
850
        editable=False)
851
852
    animal = models.ForeignKey(
853
        'Animal',
854
        on_delete=models.CASCADE)
855
856
    protocol = models.CharField(max_length=255, blank=True, null=True)
857
858
    collection_date = models.DateField(
859
        blank=True,
860
        null=True,
861
        help_text='example: 2019-04-01')
862
    collection_place_latitude = models.FloatField(blank=True, null=True)
863
    collection_place_longitude = models.FloatField(blank=True, null=True)
864
    collection_place = models.CharField(max_length=255, blank=True, null=True)
865
866
    # accuracy field (enum)
867
    collection_place_accuracy = models.SmallIntegerField(
868
        choices=[x.value for x in ACCURACIES],
869
        help_text='example: unknown accuracy level, country level',
870
        null=False,
871
        blank=False,
872
        default=MISSING)
873
874
    # using a constraint for organism (DictUberon)
875
    organism_part = models.ForeignKey(
876
        'DictUberon',
877
        null=True,
878
        on_delete=models.PROTECT)
879
880
    # using a constraint for developmental stage (DictStage)
881
    developmental_stage = models.ForeignKey(
882
        'DictStage',
883
        null=True,
884
        blank=True,
885
        on_delete=models.PROTECT)
886
887
    physiological_stage = models.CharField(
888
            max_length=255,
889
            blank=True,
890
            null=True)
891
892
    animal_age_at_collection = models.IntegerField(
893
        null=True,
894
        blank=True)
895
896
    animal_age_at_collection_units = models.SmallIntegerField(
897
        choices=[x.value for x in TIME_UNITS],
898
        help_text='example: years',
899
        null=True,
900
        blank=True)
901
902
    availability = models.CharField(max_length=255, blank=True, null=True)
903
904
    storage = models.CharField(max_length=255, blank=True, null=True)
905
906
    storage_processing = models.CharField(
907
            max_length=255,
908
            blank=True,
909
            null=True)
910
911
    preparation_interval = models.IntegerField(blank=True, null=True)
912
913
    owner = models.ForeignKey(
914
        User,
915
        related_name='samples',
916
        on_delete=models.CASCADE)
917
918
    @property
919
    def specie(self):
920
        return self.animal.breed.specie
921
922
    @property
923
    def biosample_alias(self):
924
        return 'IMAGES{0:09d}'.format(self.id)
925
926
    def get_attributes(self):
927
        """Return attributes like biosample needs"""
928
929
        attributes = super().get_attributes()
930
931
        attributes["Material"] = format_attribute(
932
            value="specimen from organism", terms="OBI_0001479")
933
934
        # The data source id or alternative id of the animal from which
935
        # the sample was collected (see Animal.to_biosample())
936
        attributes['Derived from'] = format_attribute(
937
            value=self.animal.name.name)
938
939
        # a datetime object should be not be converted in string here,
940
        # otherwise will not be filtered if NULL
941
        attributes['Collection date'] = format_attribute(
942
            value=self.collection_date, units="YYYY-MM-DD")
943
944
        attributes['Collection place'] = format_attribute(
945
            value=self.collection_place)
946
947
        attributes["Collection place longitude"] = format_attribute(
948
            value=self.collection_place_longitude,
949
            units="decimal degrees")
950
951
        attributes["Collection place latitude"] = format_attribute(
952
            value=self.collection_place_latitude,
953
            units="decimal degrees")
954
955
        attributes["Collection place accuracy"] = format_attribute(
956
            value=self.get_collection_place_accuracy_display())
957
958
        # this will point to a correct term dictionary table
959
        if self.organism_part:
960
            attributes['Organism part'] = self.organism_part.format_attribute()
961
962
        if self.developmental_stage:
963
            attributes['Developmental stage'] = \
964
                self.developmental_stage.format_attribute()
965
966
        attributes['Physiological stage'] = format_attribute(
967
            value=self.physiological_stage)
968
969
        attributes['Animal age at collection'] = format_attribute(
970
            value=self.animal_age_at_collection,
971
            units=self.get_animal_age_at_collection_units_display())
972
973
        attributes['Availability'] = format_attribute(
974
            value=self.availability)
975
976
        attributes['Sample storage'] = format_attribute(
977
            value=self.storage)
978
979
        attributes['Sample storage processing'] = format_attribute(
980
            value=self.storage_processing)
981
982
        attributes['Sampling to preparation interval'] = format_attribute(
983
            value=self.preparation_interval)
984
985
        # filter out empty values
986
        attributes = {k: v for k, v in attributes.items() if v is not None}
987
988
        return attributes
989
990
    def to_biosample(self, release_date=None):
991
        """get a json from sample for biosample submission"""
992
993
        # call methods defined in BioSampleMixin and get result
994
        # with USI mandatory keys and attributes
995
        result = super().to_biosample(release_date)
996
997
        # define relationship to the animal where this sample come from
998
        result['sampleRelationships'] = [self.animal.get_relationship()]
999
1000
        return result
1001
1002
    def get_absolute_url(self):
1003
        return reverse("samples:detail", kwargs={"pk": self.pk})
1004
1005
1006
class Person(BaseMixin, models.Model):
1007
    user = models.OneToOneField(User, on_delete=models.CASCADE)
1008
    initials = models.CharField(max_length=255, blank=True, null=True)
1009
1010
    # HINT: with a OneToOneField relation, there will be only one user for
1011
    # each organization
1012
    affiliation = models.ForeignKey(
1013
        'Organization',
1014
        null=True,
1015
        on_delete=models.PROTECT,
1016
        help_text="The institution you belong to")
1017
1018
    # last_name, first_name and email come from User model
1019
1020
    role = models.ForeignKey(
1021
        'DictRole',
1022
        on_delete=models.PROTECT,
1023
        null=True)
1024
1025
    def __str__(self):
1026
        return "{name} {surname} ({affiliation})".format(
1027
                name=self.user.first_name,
1028
                surname=self.user.last_name,
1029
                affiliation=self.affiliation)
1030
1031
1032
class Organization(BaseMixin, models.Model):
1033
    # id = models.IntegerField(primary_key=True)  # AutoField?
1034
    name = models.CharField(max_length=255)
1035
    address = models.CharField(
1036
        max_length=255, blank=True, null=True,
1037
        help_text='One line, comma separated')
1038
1039
    country = models.ForeignKey(
1040
        'DictCountry',
1041
        on_delete=models.PROTECT)
1042
1043
    URI = models.URLField(
1044
        max_length=500, blank=True, null=True,
1045
        help_text='Web site')
1046
1047
    role = models.ForeignKey(
1048
        'DictRole',
1049
        on_delete=models.PROTECT)
1050
1051
    def __str__(self):
1052
        return "%s (%s)" % (self.name, self.country.label)
1053
1054
    class Meta:
1055
        ordering = ['name', 'country']
1056
1057
1058
class Publication(BaseMixin, models.Model):
1059
    # this is a non mandatory fields in ruleset
1060
    doi = models.CharField(
1061
        max_length=255,
1062
        help_text='Valid Digital Object Identifier')
1063
1064
    def __str__(self):
1065
        return self.doi
1066
1067
1068
class Ontology(BaseMixin, models.Model):
1069
    library_name = models.CharField(
1070
        max_length=255,
1071
        help_text='Each value must be unique',
1072
        unique=True)
1073
1074
    library_uri = models.URLField(
1075
        max_length=500, blank=True, null=True,
1076
        help_text='Each value must be unique ' +
1077
                  'and with a valid URL')
1078
1079
    comment = models.CharField(
1080
            max_length=255, blank=True, null=True)
1081
1082
    def __str__(self):
1083
        return self.library_name
1084
1085
    class Meta:
1086
        verbose_name_plural = "ontologies"
1087
1088
1089
class Submission(BaseMixin, models.Model):
1090
    title = models.CharField(
1091
        "Submission title",
1092
        max_length=255,
1093
        help_text='Example: Roslin Sheep Atlas')
1094
1095
    project = models.CharField(
1096
        max_length=25,
1097
        default="IMAGE",
1098
        editable=False)
1099
1100
    description = models.CharField(
1101
        max_length=255,
1102
        help_text='Example: The Roslin Institute ' +
1103
                  'Sheep Gene Expression Atlas Project')
1104
1105
    # gene bank fields
1106
    gene_bank_name = models.CharField(
1107
        max_length=255,
1108
        blank=False,
1109
        null=False,
1110
        help_text='example: CryoWeb')
1111
1112
    gene_bank_country = models.ForeignKey(
1113
        'DictCountry',
1114
        on_delete=models.PROTECT)
1115
1116
    # datasource field
1117
    datasource_type = models.SmallIntegerField(
1118
        "Data source type",
1119
        choices=[x.value for x in DATA_TYPES],
1120
        help_text='example: CryoWeb')
1121
1122
    datasource_version = models.CharField(
1123
        "Data source version",
1124
        max_length=255,
1125
        blank=False,
1126
        null=False,
1127
        help_text='examples: "2018-04-27", "version 1.5"')
1128
1129
    organization = models.ForeignKey(
1130
        'Organization',
1131
        on_delete=models.PROTECT,
1132
        help_text="Who owns the data")
1133
1134
    # custom fields for datasource
1135
    upload_dir = 'data_source/'
1136
1137
    # File will be stored to PROTECTED_MEDIA_ROOT + upload_to
1138
    # https://gist.github.com/cobusc/ea1d01611ef05dacb0f33307e292abf4
1139
    uploaded_file = ProtectedFileField(upload_to=upload_dir)
1140
1141
    # when submission is created
1142
    created_at = models.DateTimeField(auto_now_add=True)
1143
1144
    # https://simpleisbetterthancomplex.com/tips/2016/05/23/django-tip-4-automatic-datetime-fields.html
1145
    updated_at = models.DateTimeField(auto_now=True)
1146
1147
    # a column to track submission status
1148
    status = models.SmallIntegerField(
1149
            choices=[x.value for x in STATUSES],
1150
            help_text='example: Waiting',
1151
            default=WAITING)
1152
1153
    # a field to track errors in UID loading. Should be blank if no errors
1154
    # are found
1155
    message = models.TextField(
1156
        null=True,
1157
        blank=True)
1158
1159
    # track biosample submission id in a field
1160
    # HINT: if I update a completed submision, shuold I track the
1161
    # last submission id?
1162
    biosample_submission_id = models.CharField(
1163
        max_length=255,
1164
        blank=True,
1165
        null=True,
1166
        db_index=True,
1167
        unique=True,
1168
        help_text='Biosample submission id')
1169
1170
    owner = models.ForeignKey(
1171
        User,
1172
        related_name='submissions',
1173
        on_delete=models.CASCADE)
1174
1175
    class Meta:
1176
        # HINT: can I put two files for my cryoweb instance? May they have two
1177
        # different version
1178
        unique_together = ((
1179
            "gene_bank_name",
1180
            "gene_bank_country",
1181
            "datasource_type",
1182
            "datasource_version",
1183
            "owner"),)
1184
1185
    def __str__(self):
1186
        return "%s (%s, %s)" % (
1187
            self.gene_bank_name,
1188
            self.gene_bank_country.label,
1189
            self.datasource_version
1190
        )
1191
1192
    def get_uploaded_file_basename(self):
1193
        return os.path.basename(self.uploaded_file.name)
1194
1195
    def get_uploaded_file_path(self):
1196
        """Return uploaded file path in docker container"""
1197
1198
        # this is the full path in docker container
1199
        fullpath = self.uploaded_file.file
1200
1201
        # get a string and quote fullpath
1202
        return shlex.quote(str(fullpath))
1203
1204
    def get_absolute_url(self):
1205
        return reverse("submissions:detail", kwargs={"pk": self.pk})
1206
1207
    def __can_I(self, names):
1208
        """Return True id self.status in statuses"""
1209
1210
        statuses = [x.value[0] for x in STATUSES if x.name in names]
1211
1212
        if self.status not in statuses:
1213
            return True
1214
1215
        else:
1216
            return False
1217
1218
    def can_edit(self):
1219
        """Returns True if I can edit a submission"""
1220
1221
        names = ['waiting', 'submitted']
1222
1223
        return self.__can_I(names)
1224
1225
    def can_validate(self):
1226
        names = ['error', 'waiting', 'submitted', 'completed']
1227
1228
        return self.__can_I(names)
1229
1230
    def can_submit(self):
1231
        names = ['ready']
1232
1233
        # this is the opposite of self.__can_I
1234
        statuses = [x.value[0] for x in STATUSES if x.name in names]
1235
1236
        # self.status need to be in statuses for submitting
1237
        if self.status in statuses:
1238
            return True
1239
1240
        else:
1241
            return False
1242
1243
1244
# --- Custom functions
1245
1246
1247
# https://simpleisbetterthancomplex.com/tutorial/2016/07/22/how-to-extend-django-user-model.html#onetoone
1248
# we will now define signals so our Person model will be automatically
1249
# created/updated when we create/update User instances.
1250
# Basically we are hooking the create_user_person and save_user_person
1251
# methods to the User model, whenever a save event occurs. This kind of signal
1252
# is called post_save.
1253
# TODO: add default values when creating a superuser
1254
@receiver(post_save, sender=User)
1255
def create_user_person(sender, instance, created, **kwargs):
1256
    if created:
1257
        Person.objects.create(user=instance)
1258
1259
1260
@receiver(post_save, sender=User)
1261
def save_user_person(sender, instance, **kwargs):
1262
    instance.person.save()
1263
1264
1265
# A method to truncate database
1266
def truncate_database():
1267
    """Truncate image database"""
1268
1269
    logger.warning("Truncating ALL image tables")
1270
1271
    # call each class and truncate its table by calling truncate method
1272
    Animal.truncate()
1273
    DictBreed.truncate()
1274
    DictCountry.truncate()
1275
    DictRole.truncate()
1276
    DictSex.truncate()
1277
    DictSpecie.truncate()
1278
    DictUberon.truncate()
1279
    DictStage.truncate()
1280
    Name.truncate()
1281
    Ontology.truncate()
1282
    Organization.truncate()
1283
    Person.truncate()
1284
    Publication.truncate()
1285
    Sample.truncate()
1286
    Submission.truncate()
1287
1288
    logger.warning("All cryoweb tables were truncated")
1289
1290
1291
def truncate_filled_tables():
1292
    """Truncate filled tables by import processes"""
1293
1294
    logger.warning("Truncating filled tables tables")
1295
1296
    # call each class and truncate its table by calling truncate method
1297
    Animal.truncate()
1298
    Name.truncate()
1299
    Publication.truncate()
1300
    Sample.truncate()
1301
    Submission.truncate()
1302
1303
    logger.warning("All filled tables were truncated")
1304
1305
1306
def uid_report(user):
1307
    """Performs a statistic on UID database to find issues. require user as
1308
    request.user"""
1309
1310
    report = {}
1311
1312
    # get n_of_animals
1313
    report['n_of_animals'] = Animal.objects.filter(
1314
        owner=user).count()
1315
1316
    # get n_of_samples
1317
    report['n_of_samples'] = Sample.objects.filter(
1318
        owner=user).count()
1319
1320
    # HINT: have they sense in a per user statistic?
1321
1322
    # check breeds without ontologies
1323
    breed = DictBreed.objects.filter(mapped_breed_term=None)
1324
    report['breeds_without_ontology'] = breed.count()
1325
1326
    # check countries without ontology
1327
    country = DictCountry.objects.filter(term=None)
1328
    report['countries_without_ontology'] = country.count()
1329
1330
    # check species without ontology
1331
    species = DictSpecie.objects.filter(term=None)
1332
    report['species_without_ontology'] = species.count()
1333
1334
    return report
1335
1336
1337
# A method to discover is image database has data or not
1338
def db_has_data():
1339
    # Test only tables I read data to fill UID
1340
    if (Animal.objects.exists() or Sample.objects.exists() or
1341
            Name.objects.exists()):
1342
        return True
1343
1344
    else:
1345
        return False
1346