1
|
|
|
|
2
|
|
|
import logging |
3
|
|
|
import os |
4
|
|
|
import shlex |
5
|
|
|
|
6
|
|
|
from django.contrib.auth.models import User |
7
|
|
|
from django.db import models |
8
|
|
|
from django.db.models import Func, Value, F |
9
|
|
|
from django.db.models.signals import post_save |
10
|
|
|
from django.dispatch import receiver |
11
|
|
|
from django.urls import reverse |
12
|
|
|
|
13
|
|
|
from common.fields import ProtectedFileField |
14
|
|
|
from common.constants import ( |
15
|
|
|
OBO_URL, STATUSES, CONFIDENCES, NAME_STATUSES, ACCURACIES, WAITING, LOADED, |
16
|
|
|
MISSING, DATA_TYPES, TIME_UNITS) |
17
|
|
|
|
18
|
|
|
from .helpers import format_attribute |
19
|
|
|
from .mixins import BaseMixin, BioSampleMixin |
20
|
|
|
|
21
|
|
|
# Get an instance of a logger |
22
|
|
|
logger = logging.getLogger(__name__) |
23
|
|
|
|
24
|
|
|
|
25
|
|
|
class Replace(Func): |
26
|
|
|
function = 'REPLACE' |
27
|
|
|
|
28
|
|
|
|
29
|
|
|
# --- Abstract classes |
30
|
|
|
|
31
|
|
|
|
32
|
|
|
# helper classes |
33
|
|
|
class DictBase(BaseMixin, models.Model): |
34
|
|
|
""" |
35
|
|
|
Abstract class to be inherited to all dictionary tables. It models fields |
36
|
|
|
like ``label`` (the revised term like submitter or blood) and |
37
|
|
|
``term`` (the ontology id as the final part of the URI link) |
38
|
|
|
|
39
|
|
|
The fixed part of the URI could be customized from :py:class:`Ontology` |
40
|
|
|
by setting ``library_name`` class attribute accordingly:: |
41
|
|
|
|
42
|
|
|
class DictRole(DictBase): |
43
|
|
|
library_name = 'EFO' |
44
|
|
|
""" |
45
|
|
|
|
46
|
|
|
library_name = None |
47
|
|
|
|
48
|
|
|
# if not defined, this table will have an own primary key |
49
|
|
|
label = models.CharField( |
50
|
|
|
max_length=255, |
51
|
|
|
blank=False, |
52
|
|
|
help_text="Example: submitter") |
53
|
|
|
|
54
|
|
|
term = models.CharField( |
55
|
|
|
max_length=255, |
56
|
|
|
blank=False, |
57
|
|
|
null=True, |
58
|
|
|
help_text="Example: EFO_0001741") |
59
|
|
|
|
60
|
|
|
class Meta: |
61
|
|
|
# Abstract base classes are useful when you want to put some common |
62
|
|
|
# information into a number of other models |
63
|
|
|
abstract = True |
64
|
|
|
|
65
|
|
|
def __str__(self): |
66
|
|
|
return "{label} ({term})".format( |
67
|
|
|
label=self.label, |
68
|
|
|
term=self.term) |
69
|
|
|
|
70
|
|
|
def format_attribute(self): |
71
|
|
|
""" |
72
|
|
|
Format an object instance as a dictionary used by biosample, for |
73
|
|
|
example:: |
74
|
|
|
|
75
|
|
|
[{ |
76
|
|
|
'value': 'submitter', |
77
|
|
|
'terms': [{'url': 'http://www.ebi.ac.uk/efo/EFO_0001741'}] |
78
|
|
|
}] |
79
|
|
|
|
80
|
|
|
the fixed part of URI link is defined by ``library_name`` class |
81
|
|
|
attribute |
82
|
|
|
""" |
83
|
|
|
|
84
|
|
|
if self.library_name is None: |
85
|
|
|
logger.warning("library_name not defined") |
86
|
|
|
library_uri = OBO_URL |
87
|
|
|
|
88
|
|
|
else: |
89
|
|
|
library = Ontology.objects.get(library_name=self.library_name) |
90
|
|
|
library_uri = library.library_uri |
91
|
|
|
|
92
|
|
|
return format_attribute( |
93
|
|
|
value=self.label, |
94
|
|
|
library_uri=library_uri, |
95
|
|
|
terms=self.term) |
96
|
|
|
|
97
|
|
|
|
98
|
|
|
class Confidence(BaseMixin, models.Model): |
99
|
|
|
""" |
100
|
|
|
Abstract class which add :ref:`confidence <Common confidences>` |
101
|
|
|
to models |
102
|
|
|
""" |
103
|
|
|
|
104
|
|
|
# confidence field (enum) |
105
|
|
|
confidence = models.SmallIntegerField( |
106
|
|
|
choices=[x.value for x in CONFIDENCES], |
107
|
|
|
help_text='example: Manually Curated', |
108
|
|
|
null=True) |
109
|
|
|
|
110
|
|
|
class Meta: |
111
|
|
|
# Abstract base classes are useful when you want to put some common |
112
|
|
|
# information into a number of other models |
113
|
|
|
abstract = True |
114
|
|
|
|
115
|
|
|
|
116
|
|
|
# --- dictionary tables |
117
|
|
|
|
118
|
|
|
|
119
|
|
|
class DictRole(DictBase): |
120
|
|
|
"""A class to model roles defined as childs of |
121
|
|
|
http://www.ebi.ac.uk/efo/EFO_0002012""" |
122
|
|
|
|
123
|
|
|
library_name = 'EFO' |
124
|
|
|
|
125
|
|
|
class Meta: |
126
|
|
|
# db_table will be <app_name>_<classname> |
127
|
|
|
verbose_name = "role" |
128
|
|
|
unique_together = (("label", "term"),) |
129
|
|
|
|
130
|
|
|
|
131
|
|
|
class DictCountry(DictBase, Confidence): |
132
|
|
|
"""A class to model contries defined by NCI Thesaurus OBO Edition |
133
|
|
|
https://www.ebi.ac.uk/ols/ontologies/ncit""" |
134
|
|
|
|
135
|
|
|
library_name = 'NCIT' |
136
|
|
|
|
137
|
|
|
class Meta: |
138
|
|
|
# db_table will be <app_name>_<classname> |
139
|
|
|
verbose_name = "country" |
140
|
|
|
verbose_name_plural = "countries" |
141
|
|
|
unique_together = (("label", "term"),) |
142
|
|
|
ordering = ['label'] |
143
|
|
|
|
144
|
|
|
|
145
|
|
|
class DictSex(DictBase): |
146
|
|
|
"""A class to model sex as defined in PATO""" |
147
|
|
|
|
148
|
|
|
library_name = "PATO" |
149
|
|
|
|
150
|
|
|
class Meta: |
151
|
|
|
verbose_name = 'sex' |
152
|
|
|
verbose_name_plural = 'sex' |
153
|
|
|
unique_together = (("label", "term"),) |
154
|
|
|
|
155
|
|
|
|
156
|
|
|
class DictUberon(DictBase, Confidence): |
157
|
|
|
"""A class to model anatomies modeled in uberon""" |
158
|
|
|
|
159
|
|
|
library_name = "UBERON" |
160
|
|
|
|
161
|
|
|
class Meta: |
162
|
|
|
verbose_name = 'organism part' |
163
|
|
|
unique_together = (("label", "term"),) |
164
|
|
|
|
165
|
|
|
|
166
|
|
|
class DictStage(DictBase, Confidence): |
167
|
|
|
"""A class to developmental stages defined as descendants of |
168
|
|
|
descendants of EFO_0000399""" |
169
|
|
|
|
170
|
|
|
library_name = 'EFO' |
171
|
|
|
|
172
|
|
|
class Meta: |
173
|
|
|
# db_table will be <app_name>_<classname> |
174
|
|
|
verbose_name = "developmental stage" |
175
|
|
|
unique_together = (("label", "term"),) |
176
|
|
|
|
177
|
|
|
|
178
|
|
|
class DictSpecie(DictBase, Confidence): |
179
|
|
|
"""A class to model species defined by NCBI organismal classification |
180
|
|
|
http://www.ebi.ac.uk/ols/ontologies/ncbitaxon""" |
181
|
|
|
|
182
|
|
|
library_name = "NCBITaxon" |
183
|
|
|
|
184
|
|
|
@property |
185
|
|
|
def taxon_id(self): |
186
|
|
|
if not self.term or self.term == '': |
187
|
|
|
return None |
188
|
|
|
|
189
|
|
|
return int(self.term.split("_")[-1]) |
190
|
|
|
|
191
|
|
|
class Meta: |
192
|
|
|
# db_table will be <app_name>_<classname> |
193
|
|
|
verbose_name = "specie" |
194
|
|
|
unique_together = (("label", "term"),) |
195
|
|
|
|
196
|
|
|
@classmethod |
197
|
|
|
def get_by_synonym(cls, synonym, language): |
198
|
|
|
"""return an instance by synonym in supplied language or default one""" |
199
|
|
|
|
200
|
|
|
# get a queryset with speciesynonym |
201
|
|
|
qs = cls.objects.prefetch_related('speciesynonym_set') |
202
|
|
|
|
203
|
|
|
# annotate queryset by removing spaces from speciesynonym word |
204
|
|
|
qs = qs.annotate( |
205
|
|
|
new_word=Replace('speciesynonym__word', Value(" "), Value("")), |
206
|
|
|
language=F('speciesynonym__language__label')) |
207
|
|
|
|
208
|
|
|
# now remove spaces from synonym |
209
|
|
|
synonym = synonym.replace(" ", "") |
210
|
|
|
|
211
|
|
|
try: |
212
|
|
|
specie = qs.get( |
213
|
|
|
new_word=synonym, |
214
|
|
|
language=language) |
215
|
|
|
|
216
|
|
|
except cls.DoesNotExist: |
217
|
|
|
specie = qs.get( |
218
|
|
|
new_word=synonym, |
219
|
|
|
language="United Kingdom") |
220
|
|
|
|
221
|
|
|
return specie |
222
|
|
|
|
223
|
|
|
|
224
|
|
|
class DictBreed(Confidence): |
225
|
|
|
"""A class to deal with breed objects and their ontologies""" |
226
|
|
|
|
227
|
|
|
library_name = "LBO" |
228
|
|
|
|
229
|
|
|
# this was the description field in cryoweb v_breeds_species tables |
230
|
|
|
supplied_breed = models.CharField(max_length=255, blank=False) |
231
|
|
|
mapped_breed = models.CharField(max_length=255, blank=False, null=True) |
232
|
|
|
|
233
|
|
|
mapped_breed_term = models.CharField( |
234
|
|
|
max_length=255, |
235
|
|
|
blank=False, |
236
|
|
|
null=True, |
237
|
|
|
help_text="Example: LBO_0000347") |
238
|
|
|
|
239
|
|
|
# using a constraint for country. |
240
|
|
|
country = models.ForeignKey( |
241
|
|
|
'DictCountry', |
242
|
|
|
on_delete=models.PROTECT) |
243
|
|
|
|
244
|
|
|
# using a constraint for specie |
245
|
|
|
specie = models.ForeignKey( |
246
|
|
|
'DictSpecie', |
247
|
|
|
on_delete=models.PROTECT) |
248
|
|
|
|
249
|
|
|
class Meta: |
250
|
|
|
verbose_name = 'Breed' |
251
|
|
|
unique_together = (("supplied_breed", "specie", "country"),) |
252
|
|
|
|
253
|
|
|
def __str__(self): |
254
|
|
|
return "{supplied} ({mapped}, {specie})".format( |
255
|
|
|
supplied=self.supplied_breed, |
256
|
|
|
mapped=self.mapped_breed, |
257
|
|
|
specie=self.specie.label) |
258
|
|
|
|
259
|
|
|
def format_attribute(self): |
260
|
|
|
"""Format mapped_breed attribute (with its ontology). Return None if |
261
|
|
|
no mapped_breed""" |
262
|
|
|
|
263
|
|
|
if not self.mapped_breed or not self.mapped_breed_term: |
264
|
|
|
return None |
265
|
|
|
|
266
|
|
|
library = Ontology.objects.get(library_name=self.library_name) |
267
|
|
|
library_uri = library.library_uri |
268
|
|
|
|
269
|
|
|
return format_attribute( |
270
|
|
|
value=self.mapped_breed, |
271
|
|
|
library_uri=library_uri, |
272
|
|
|
terms=self.mapped_breed_term) |
273
|
|
|
|
274
|
|
|
|
275
|
|
|
# --- Other tables tables |
276
|
|
|
|
277
|
|
|
|
278
|
|
|
class Name(BaseMixin, models.Model): |
279
|
|
|
"""Model UID names: define a name (sample or animal) unique for each |
280
|
|
|
data submission""" |
281
|
|
|
|
282
|
|
|
# two different animal may have the same name. Its unicity depens on |
283
|
|
|
# data source name and version |
284
|
|
|
name = models.CharField( |
285
|
|
|
max_length=255, |
286
|
|
|
blank=False, |
287
|
|
|
null=False) |
288
|
|
|
|
289
|
|
|
submission = models.ForeignKey( |
290
|
|
|
'Submission', |
291
|
|
|
db_index=True, |
292
|
|
|
related_name='name_set', |
293
|
|
|
on_delete=models.CASCADE) |
294
|
|
|
|
295
|
|
|
# This need to be assigned after submission |
296
|
|
|
# HINT: this column should be UNIQUE? |
297
|
|
|
biosample_id = models.CharField(max_length=255, blank=True, null=True) |
298
|
|
|
|
299
|
|
|
# '+' instructs Django that we don’t need this reverse relationship |
300
|
|
|
owner = models.ForeignKey( |
301
|
|
|
User, |
302
|
|
|
related_name='+', |
303
|
|
|
on_delete=models.CASCADE) |
304
|
|
|
|
305
|
|
|
# a column to track submission status |
306
|
|
|
status = models.SmallIntegerField( |
307
|
|
|
choices=[x.value for x in STATUSES if x.name in NAME_STATUSES], |
308
|
|
|
help_text='example: Submitted', |
309
|
|
|
default=LOADED) |
310
|
|
|
|
311
|
|
|
last_changed = models.DateTimeField( |
312
|
|
|
auto_now_add=True, |
313
|
|
|
blank=True, |
314
|
|
|
null=True) |
315
|
|
|
|
316
|
|
|
last_submitted = models.DateTimeField( |
317
|
|
|
blank=True, |
318
|
|
|
null=True) |
319
|
|
|
|
320
|
|
|
publication = models.ForeignKey( |
321
|
|
|
'Publication', |
322
|
|
|
null=True, |
323
|
|
|
blank=True, |
324
|
|
|
related_name='name_set', |
325
|
|
|
on_delete=models.SET_NULL) |
326
|
|
|
|
327
|
|
|
class Meta: |
328
|
|
|
unique_together = (("name", "submission"),) |
329
|
|
|
|
330
|
|
|
def __str__(self): |
331
|
|
|
# HINT: should I return biosampleid if defined? |
332
|
|
|
return "%s (Submission: %s)" % (self.name, self.submission_id) |
333
|
|
|
|
334
|
|
|
|
335
|
|
|
class Animal(BioSampleMixin, models.Model): |
336
|
|
|
""" |
337
|
|
|
Class to model Animal (Organism). Inherits from :py:class:`BioSampleMixin`, |
338
|
|
|
related to :py:class:`Name` through ``OneToOne`` relationship to model |
339
|
|
|
Animal name (Data source id), and with the same relationship to model |
340
|
|
|
``mother`` and ``father`` of such animal. In case that parents are unknown, |
341
|
|
|
could be linked with Unkwnon animals for cryoweb data or doens't have |
342
|
|
|
relationship.Linked to :py:class:`DictBreed` dictionary |
343
|
|
|
table to model info on species and breed. Linked to |
344
|
|
|
:py:class:`Sample` to model Samples (Specimen from organims):: |
345
|
|
|
|
346
|
|
|
from image_app.models import Animal |
347
|
|
|
|
348
|
|
|
# get animal using primary key |
349
|
|
|
animal = Animal.objects.get(pk=1) |
350
|
|
|
|
351
|
|
|
# get animal name |
352
|
|
|
data_source_id = animal.name.name |
353
|
|
|
|
354
|
|
|
# get animal's parents |
355
|
|
|
mother = animal.mother |
356
|
|
|
father = animal.father |
357
|
|
|
|
358
|
|
|
# get breed and species info |
359
|
|
|
print(animal.breed.supplied_breed) |
360
|
|
|
print(animal.breed.specie.label) |
361
|
|
|
|
362
|
|
|
# get all samples (specimen) for this animals |
363
|
|
|
samples = animal.sample_set.all() |
364
|
|
|
""" |
365
|
|
|
|
366
|
|
|
# an animal name has a entry in name table |
367
|
|
|
name = models.OneToOneField( |
368
|
|
|
'Name', |
369
|
|
|
on_delete=models.CASCADE) |
370
|
|
|
|
371
|
|
|
# alternative id will store the internal id in data source |
372
|
|
|
alternative_id = models.CharField(max_length=255, blank=True, null=True) |
373
|
|
|
|
374
|
|
|
description = models.CharField(max_length=255, blank=True, null=True) |
375
|
|
|
|
376
|
|
|
# HINT: link to a term list table? |
377
|
|
|
material = models.CharField( |
378
|
|
|
max_length=255, |
379
|
|
|
default="Organism", |
380
|
|
|
editable=False) |
381
|
|
|
|
382
|
|
|
breed = models.ForeignKey( |
383
|
|
|
'DictBreed', |
384
|
|
|
db_index=True, |
385
|
|
|
on_delete=models.PROTECT) |
386
|
|
|
|
387
|
|
|
# species is in DictBreed table |
388
|
|
|
|
389
|
|
|
# using a constraint for sex |
390
|
|
|
sex = models.ForeignKey( |
391
|
|
|
'DictSex', |
392
|
|
|
null=True, |
393
|
|
|
on_delete=models.PROTECT) |
394
|
|
|
|
395
|
|
|
# check that father and mother are defined using Foreign Keys |
396
|
|
|
# HINT: mother and father are not mandatory in all datasource |
397
|
|
|
father = models.ForeignKey( |
398
|
|
|
'Name', |
399
|
|
|
on_delete=models.CASCADE, |
400
|
|
|
null=True, |
401
|
|
|
related_name='father_of') |
402
|
|
|
|
403
|
|
|
mother = models.ForeignKey( |
404
|
|
|
'Name', |
405
|
|
|
on_delete=models.CASCADE, |
406
|
|
|
null=True, |
407
|
|
|
related_name='mother_of') |
408
|
|
|
|
409
|
|
|
birth_date = models.DateField( |
410
|
|
|
blank=True, |
411
|
|
|
null=True, |
412
|
|
|
help_text='example: 2019-04-01') |
413
|
|
|
|
414
|
|
|
# TODO: need to set this value? How? |
415
|
|
|
birth_location = models.CharField( |
416
|
|
|
max_length=255, |
417
|
|
|
blank=True, |
418
|
|
|
null=True) |
419
|
|
|
|
420
|
|
|
birth_location_latitude = models.FloatField(blank=True, null=True) |
421
|
|
|
birth_location_longitude = models.FloatField(blank=True, null=True) |
422
|
|
|
|
423
|
|
|
# accuracy field (enum) |
424
|
|
|
birth_location_accuracy = models.SmallIntegerField( |
425
|
|
|
choices=[x.value for x in ACCURACIES], |
426
|
|
|
help_text='example: unknown accuracy level, country level', |
427
|
|
|
null=False, |
428
|
|
|
blank=False, |
429
|
|
|
default=MISSING) |
430
|
|
|
|
431
|
|
|
owner = models.ForeignKey( |
432
|
|
|
User, |
433
|
|
|
related_name='animals', |
434
|
|
|
on_delete=models.CASCADE) |
435
|
|
|
|
436
|
|
|
@property |
437
|
|
|
def specie(self): |
438
|
|
|
return self.breed.specie |
439
|
|
|
|
440
|
|
|
@property |
441
|
|
|
def biosample_alias(self): |
442
|
|
|
return 'IMAGEA{0:09d}'.format(self.id) |
443
|
|
|
|
444
|
|
|
def get_attributes(self): |
445
|
|
|
"""Return attributes like biosample needs""" |
446
|
|
|
|
447
|
|
|
attributes = super().get_attributes() |
448
|
|
|
|
449
|
|
|
attributes["Material"] = format_attribute( |
450
|
|
|
value="organism", terms="OBI_0100026") |
451
|
|
|
|
452
|
|
|
# TODO: how to model derived from (mother/father)? |
453
|
|
|
|
454
|
|
|
attributes['Supplied breed'] = format_attribute( |
455
|
|
|
value=self.breed.supplied_breed) |
456
|
|
|
|
457
|
|
|
# HINT: Ideally, I could retrieve an ontology id for countries |
458
|
|
|
attributes['EFABIS Breed country'] = format_attribute( |
459
|
|
|
value=self.breed.country.label) |
460
|
|
|
|
461
|
|
|
attributes['Mapped breed'] = self.breed.format_attribute() |
462
|
|
|
|
463
|
|
|
attributes['Sex'] = self.sex.format_attribute() |
464
|
|
|
|
465
|
|
|
# a datetime object should be not be converted in string here, |
466
|
|
|
# otherwise will not be filtered if NULL |
467
|
|
|
attributes['Birth date'] = format_attribute( |
468
|
|
|
value=self.birth_date, units="YYYY-MM-DD") |
469
|
|
|
|
470
|
|
|
attributes["Birth location"] = format_attribute( |
471
|
|
|
value=self.birth_location) |
472
|
|
|
|
473
|
|
|
attributes["Birth location longitude"] = format_attribute( |
474
|
|
|
value=self.birth_location_longitude, |
475
|
|
|
units="decimal degrees") |
476
|
|
|
|
477
|
|
|
attributes["Birth location latitude"] = format_attribute( |
478
|
|
|
value=self.birth_location_latitude, |
479
|
|
|
units="decimal degrees") |
480
|
|
|
|
481
|
|
|
attributes["Birth location accuracy"] = format_attribute( |
482
|
|
|
value=self.get_birth_location_accuracy_display()) |
483
|
|
|
|
484
|
|
|
# filter out empty values |
485
|
|
|
attributes = {k: v for k, v in attributes.items() if v is not None} |
486
|
|
|
|
487
|
|
|
return attributes |
488
|
|
|
|
489
|
|
|
def get_relationship(self): |
490
|
|
|
"""Get a relationship to this animal (call this method from a related |
491
|
|
|
object to get a connection to this element)""" |
492
|
|
|
|
493
|
|
|
# if animal is already uploaded I will use accession as |
494
|
|
|
# relationship key. This biosample id could be tested in validation |
495
|
|
|
if self.biosample_id and self.biosample_id != '': |
496
|
|
|
return { |
497
|
|
|
"accession": self.biosample_id, |
498
|
|
|
"relationshipNature": "derived from", |
499
|
|
|
} |
500
|
|
|
else: |
501
|
|
|
return { |
502
|
|
|
"alias": self.biosample_alias, |
503
|
|
|
"relationshipNature": "derived from", |
504
|
|
|
} |
505
|
|
|
|
506
|
|
|
def get_father_relationship(self): |
507
|
|
|
"""Get a relationship with father if possible""" |
508
|
|
|
|
509
|
|
|
# get father of this animal |
510
|
|
|
if self.father is None: |
511
|
|
|
return None |
512
|
|
|
|
513
|
|
|
# cryoweb could have unkwown animals. They are Names without |
514
|
|
|
# relationships with Animal table |
515
|
|
|
if not hasattr(self.father, "animal"): |
516
|
|
|
return None |
517
|
|
|
|
518
|
|
|
else: |
519
|
|
|
return self.father.animal.get_relationship() |
520
|
|
|
|
521
|
|
|
def get_mother_relationship(self): |
522
|
|
|
"""Get a relationship with mother if possible""" |
523
|
|
|
|
524
|
|
|
# get mother of this animal |
525
|
|
|
if self.mother is None: |
526
|
|
|
return None |
527
|
|
|
|
528
|
|
|
# cryoweb could have unkwown animals. They are Names without |
529
|
|
|
# relationships with Animal table |
530
|
|
|
if not hasattr(self.mother, "animal"): |
531
|
|
|
return None |
532
|
|
|
|
533
|
|
|
else: |
534
|
|
|
return self.mother.animal.get_relationship() |
535
|
|
|
|
536
|
|
|
def to_biosample(self, release_date=None): |
537
|
|
|
"""get a json from animal for biosample submission""" |
538
|
|
|
|
539
|
|
|
# call methods defined in BioSampleMixin and get result |
540
|
|
|
# with USI mandatory keys and attributes |
541
|
|
|
result = super().to_biosample(release_date) |
542
|
|
|
|
543
|
|
|
# define relationship with mother and father (if possible) |
544
|
|
|
result['sampleRelationships'] = [] |
545
|
|
|
|
546
|
|
|
father_relationship = self.get_father_relationship() |
547
|
|
|
|
548
|
|
|
if father_relationship is not None: |
549
|
|
|
result['sampleRelationships'].append(father_relationship) |
550
|
|
|
|
551
|
|
|
mother_relationship = self.get_mother_relationship() |
552
|
|
|
|
553
|
|
|
if mother_relationship is not None: |
554
|
|
|
result['sampleRelationships'].append(mother_relationship) |
555
|
|
|
|
556
|
|
|
return result |
557
|
|
|
|
558
|
|
|
def get_absolute_url(self): |
559
|
|
|
return reverse("animals:detail", kwargs={"pk": self.pk}) |
560
|
|
|
|
561
|
|
|
|
562
|
|
|
class Sample(BioSampleMixin, models.Model): |
563
|
|
|
# a sample name has a entry in name table |
564
|
|
|
# this is a One2One foreign key |
565
|
|
|
name = models.OneToOneField( |
566
|
|
|
'Name', |
567
|
|
|
on_delete=models.CASCADE) |
568
|
|
|
|
569
|
|
|
# db_vessel in data source |
570
|
|
|
alternative_id = models.CharField(max_length=255, blank=True, null=True) |
571
|
|
|
|
572
|
|
|
description = models.CharField(max_length=255, blank=True, null=True) |
573
|
|
|
|
574
|
|
|
# HINT: link to a term list table? |
575
|
|
|
material = models.CharField( |
576
|
|
|
max_length=255, |
577
|
|
|
default="Specimen from Organism", |
578
|
|
|
editable=False) |
579
|
|
|
|
580
|
|
|
animal = models.ForeignKey( |
581
|
|
|
'Animal', |
582
|
|
|
on_delete=models.CASCADE) |
583
|
|
|
|
584
|
|
|
# HINT: should this be a protocol? |
585
|
|
|
protocol = models.CharField(max_length=255, blank=True, null=True) |
586
|
|
|
|
587
|
|
|
collection_date = models.DateField( |
588
|
|
|
blank=True, |
589
|
|
|
null=True, |
590
|
|
|
help_text='example: 2019-04-01') |
591
|
|
|
collection_place_latitude = models.FloatField(blank=True, null=True) |
592
|
|
|
collection_place_longitude = models.FloatField(blank=True, null=True) |
593
|
|
|
collection_place = models.CharField(max_length=255, blank=True, null=True) |
594
|
|
|
|
595
|
|
|
# accuracy field (enum) |
596
|
|
|
collection_place_accuracy = models.SmallIntegerField( |
597
|
|
|
choices=[x.value for x in ACCURACIES], |
598
|
|
|
help_text='example: unknown accuracy level, country level', |
599
|
|
|
null=False, |
600
|
|
|
blank=False, |
601
|
|
|
default=MISSING) |
602
|
|
|
|
603
|
|
|
# using a constraint for organism (DictUberon) |
604
|
|
|
organism_part = models.ForeignKey( |
605
|
|
|
'DictUberon', |
606
|
|
|
null=True, |
607
|
|
|
on_delete=models.PROTECT) |
608
|
|
|
|
609
|
|
|
# using a constraint for developmental stage (DictStage) |
610
|
|
|
developmental_stage = models.ForeignKey( |
611
|
|
|
'DictStage', |
612
|
|
|
null=True, |
613
|
|
|
blank=True, |
614
|
|
|
on_delete=models.PROTECT) |
615
|
|
|
|
616
|
|
|
physiological_stage = models.CharField( |
617
|
|
|
max_length=255, |
618
|
|
|
blank=True, |
619
|
|
|
null=True) |
620
|
|
|
|
621
|
|
|
animal_age_at_collection = models.IntegerField( |
622
|
|
|
null=True, |
623
|
|
|
blank=True) |
624
|
|
|
|
625
|
|
|
animal_age_at_collection_units = models.SmallIntegerField( |
626
|
|
|
choices=[x.value for x in TIME_UNITS], |
627
|
|
|
help_text='example: years', |
628
|
|
|
null=True, |
629
|
|
|
blank=True) |
630
|
|
|
|
631
|
|
|
availability = models.CharField(max_length=255, blank=True, null=True) |
632
|
|
|
|
633
|
|
|
storage = models.CharField(max_length=255, blank=True, null=True) |
634
|
|
|
|
635
|
|
|
storage_processing = models.CharField( |
636
|
|
|
max_length=255, |
637
|
|
|
blank=True, |
638
|
|
|
null=True) |
639
|
|
|
|
640
|
|
|
preparation_interval = models.IntegerField(blank=True, null=True) |
641
|
|
|
|
642
|
|
|
owner = models.ForeignKey( |
643
|
|
|
User, |
644
|
|
|
related_name='samples', |
645
|
|
|
on_delete=models.CASCADE) |
646
|
|
|
|
647
|
|
|
@property |
648
|
|
|
def specie(self): |
649
|
|
|
return self.animal.breed.specie |
650
|
|
|
|
651
|
|
|
@property |
652
|
|
|
def biosample_alias(self): |
653
|
|
|
return 'IMAGES{0:09d}'.format(self.id) |
654
|
|
|
|
655
|
|
|
def get_attributes(self): |
656
|
|
|
"""Return attributes like biosample needs""" |
657
|
|
|
|
658
|
|
|
attributes = super().get_attributes() |
659
|
|
|
|
660
|
|
|
attributes["Material"] = format_attribute( |
661
|
|
|
value="specimen from organism", terms="OBI_0001479") |
662
|
|
|
|
663
|
|
|
# The data source id or alternative id of the animal from which |
664
|
|
|
# the sample was collected (see Animal.to_biosample()) |
665
|
|
|
attributes['Derived from'] = format_attribute( |
666
|
|
|
value=self.animal.name.name) |
667
|
|
|
|
668
|
|
|
attributes["Specimen collection protocol"] = format_attribute( |
669
|
|
|
value=self.protocol) |
670
|
|
|
|
671
|
|
|
# a datetime object should be not be converted in string here, |
672
|
|
|
# otherwise will not be filtered if NULL |
673
|
|
|
attributes['Collection date'] = format_attribute( |
674
|
|
|
value=self.collection_date, units="YYYY-MM-DD") |
675
|
|
|
|
676
|
|
|
attributes['Collection place'] = format_attribute( |
677
|
|
|
value=self.collection_place) |
678
|
|
|
|
679
|
|
|
attributes["Collection place longitude"] = format_attribute( |
680
|
|
|
value=self.collection_place_longitude, |
681
|
|
|
units="decimal degrees") |
682
|
|
|
|
683
|
|
|
attributes["Collection place latitude"] = format_attribute( |
684
|
|
|
value=self.collection_place_latitude, |
685
|
|
|
units="decimal degrees") |
686
|
|
|
|
687
|
|
|
attributes["Collection place accuracy"] = format_attribute( |
688
|
|
|
value=self.get_collection_place_accuracy_display()) |
689
|
|
|
|
690
|
|
|
# this will point to a correct term dictionary table |
691
|
|
|
if self.organism_part: |
692
|
|
|
attributes['Organism part'] = self.organism_part.format_attribute() |
693
|
|
|
|
694
|
|
|
if self.developmental_stage: |
695
|
|
|
attributes['Developmental stage'] = \ |
696
|
|
|
self.developmental_stage.format_attribute() |
697
|
|
|
|
698
|
|
|
attributes['Physiological stage'] = format_attribute( |
699
|
|
|
value=self.physiological_stage) |
700
|
|
|
|
701
|
|
|
attributes['Animal age at collection'] = format_attribute( |
702
|
|
|
value=self.animal_age_at_collection, |
703
|
|
|
units=self.get_animal_age_at_collection_units_display()) |
704
|
|
|
|
705
|
|
|
attributes['Availability'] = format_attribute( |
706
|
|
|
value=self.availability) |
707
|
|
|
|
708
|
|
|
attributes['Sample storage'] = format_attribute( |
709
|
|
|
value=self.storage) |
710
|
|
|
|
711
|
|
|
attributes['Sample storage processing'] = format_attribute( |
712
|
|
|
value=self.storage_processing) |
713
|
|
|
|
714
|
|
|
attributes['Sampling to preparation interval'] = format_attribute( |
715
|
|
|
value=self.preparation_interval) |
716
|
|
|
|
717
|
|
|
# filter out empty values |
718
|
|
|
attributes = {k: v for k, v in attributes.items() if v is not None} |
719
|
|
|
|
720
|
|
|
return attributes |
721
|
|
|
|
722
|
|
|
def to_biosample(self, release_date=None): |
723
|
|
|
"""get a json from sample for biosample submission""" |
724
|
|
|
|
725
|
|
|
# call methods defined in BioSampleMixin and get result |
726
|
|
|
# with USI mandatory keys and attributes |
727
|
|
|
result = super().to_biosample(release_date) |
728
|
|
|
|
729
|
|
|
# define relationship to the animal where this sample come from |
730
|
|
|
result['sampleRelationships'] = [self.animal.get_relationship()] |
731
|
|
|
|
732
|
|
|
return result |
733
|
|
|
|
734
|
|
|
def get_absolute_url(self): |
735
|
|
|
return reverse("samples:detail", kwargs={"pk": self.pk}) |
736
|
|
|
|
737
|
|
|
|
738
|
|
|
class Person(BaseMixin, models.Model): |
739
|
|
|
user = models.OneToOneField(User, on_delete=models.CASCADE) |
740
|
|
|
initials = models.CharField(max_length=255, blank=True, null=True) |
741
|
|
|
|
742
|
|
|
# HINT: with a OneToOneField relation, there will be only one user for |
743
|
|
|
# each organization |
744
|
|
|
affiliation = models.ForeignKey( |
745
|
|
|
'Organization', |
746
|
|
|
null=True, |
747
|
|
|
on_delete=models.PROTECT, |
748
|
|
|
help_text="The institution you belong to") |
749
|
|
|
|
750
|
|
|
# last_name, first_name and email come from User model |
751
|
|
|
|
752
|
|
|
role = models.ForeignKey( |
753
|
|
|
'DictRole', |
754
|
|
|
on_delete=models.PROTECT, |
755
|
|
|
null=True) |
756
|
|
|
|
757
|
|
|
def __str__(self): |
758
|
|
|
return "{name} {surname} ({affiliation})".format( |
759
|
|
|
name=self.user.first_name, |
760
|
|
|
surname=self.user.last_name, |
761
|
|
|
affiliation=self.affiliation) |
762
|
|
|
|
763
|
|
|
|
764
|
|
|
class Organization(BaseMixin, models.Model): |
765
|
|
|
# id = models.IntegerField(primary_key=True) # AutoField? |
766
|
|
|
name = models.CharField(max_length=255) |
767
|
|
|
address = models.CharField( |
768
|
|
|
max_length=255, blank=True, null=True, |
769
|
|
|
help_text='One line, comma separated') |
770
|
|
|
|
771
|
|
|
country = models.ForeignKey( |
772
|
|
|
'DictCountry', |
773
|
|
|
on_delete=models.PROTECT) |
774
|
|
|
|
775
|
|
|
URI = models.URLField( |
776
|
|
|
max_length=500, blank=True, null=True, |
777
|
|
|
help_text='Web site') |
778
|
|
|
|
779
|
|
|
role = models.ForeignKey( |
780
|
|
|
'DictRole', |
781
|
|
|
on_delete=models.PROTECT) |
782
|
|
|
|
783
|
|
|
def __str__(self): |
784
|
|
|
return "%s (%s)" % (self.name, self.country.label) |
785
|
|
|
|
786
|
|
|
class Meta: |
787
|
|
|
ordering = ['name', 'country'] |
788
|
|
|
|
789
|
|
|
|
790
|
|
|
class Publication(BaseMixin, models.Model): |
791
|
|
|
# this is a non mandatory fields in ruleset |
792
|
|
|
doi = models.CharField( |
793
|
|
|
max_length=255, |
794
|
|
|
help_text='Valid Digital Object Identifier') |
795
|
|
|
|
796
|
|
|
def __str__(self): |
797
|
|
|
return self.doi |
798
|
|
|
|
799
|
|
|
|
800
|
|
|
class Ontology(BaseMixin, models.Model): |
801
|
|
|
library_name = models.CharField( |
802
|
|
|
max_length=255, |
803
|
|
|
help_text='Each value must be unique', |
804
|
|
|
unique=True) |
805
|
|
|
|
806
|
|
|
library_uri = models.URLField( |
807
|
|
|
max_length=500, blank=True, null=True, |
808
|
|
|
help_text='Each value must be unique ' + |
809
|
|
|
'and with a valid URL') |
810
|
|
|
|
811
|
|
|
comment = models.CharField( |
812
|
|
|
max_length=255, blank=True, null=True) |
813
|
|
|
|
814
|
|
|
def __str__(self): |
815
|
|
|
return self.library_name |
816
|
|
|
|
817
|
|
|
class Meta: |
818
|
|
|
verbose_name_plural = "ontologies" |
819
|
|
|
|
820
|
|
|
|
821
|
|
|
class Submission(BaseMixin, models.Model): |
822
|
|
|
title = models.CharField( |
823
|
|
|
"Submission title", |
824
|
|
|
max_length=255, |
825
|
|
|
help_text='Example: Roslin Sheep Atlas') |
826
|
|
|
|
827
|
|
|
project = models.CharField( |
828
|
|
|
max_length=25, |
829
|
|
|
default="IMAGE", |
830
|
|
|
editable=False) |
831
|
|
|
|
832
|
|
|
description = models.CharField( |
833
|
|
|
max_length=255, |
834
|
|
|
help_text='Example: The Roslin Institute ' + |
835
|
|
|
'Sheep Gene Expression Atlas Project') |
836
|
|
|
|
837
|
|
|
# gene bank fields |
838
|
|
|
gene_bank_name = models.CharField( |
839
|
|
|
max_length=255, |
840
|
|
|
blank=False, |
841
|
|
|
null=False, |
842
|
|
|
help_text='example: CryoWeb') |
843
|
|
|
|
844
|
|
|
gene_bank_country = models.ForeignKey( |
845
|
|
|
'DictCountry', |
846
|
|
|
on_delete=models.PROTECT) |
847
|
|
|
|
848
|
|
|
# datasource field |
849
|
|
|
datasource_type = models.SmallIntegerField( |
850
|
|
|
"Data source type", |
851
|
|
|
choices=[x.value for x in DATA_TYPES], |
852
|
|
|
help_text='example: CryoWeb') |
853
|
|
|
|
854
|
|
|
datasource_version = models.CharField( |
855
|
|
|
"Data source version", |
856
|
|
|
max_length=255, |
857
|
|
|
blank=False, |
858
|
|
|
null=False, |
859
|
|
|
help_text='examples: "2018-04-27", "version 1.5"') |
860
|
|
|
|
861
|
|
|
organization = models.ForeignKey( |
862
|
|
|
'Organization', |
863
|
|
|
on_delete=models.PROTECT, |
864
|
|
|
help_text="Who owns the data") |
865
|
|
|
|
866
|
|
|
# custom fields for datasource |
867
|
|
|
upload_dir = 'data_source/' |
868
|
|
|
|
869
|
|
|
# File will be stored to PROTECTED_MEDIA_ROOT + upload_to |
870
|
|
|
# https://gist.github.com/cobusc/ea1d01611ef05dacb0f33307e292abf4 |
871
|
|
|
uploaded_file = ProtectedFileField(upload_to=upload_dir) |
872
|
|
|
|
873
|
|
|
# when submission is created |
874
|
|
|
created_at = models.DateTimeField(auto_now_add=True) |
875
|
|
|
|
876
|
|
|
# https://simpleisbetterthancomplex.com/tips/2016/05/23/django-tip-4-automatic-datetime-fields.html |
877
|
|
|
updated_at = models.DateTimeField(auto_now=True) |
878
|
|
|
|
879
|
|
|
# a column to track submission status |
880
|
|
|
status = models.SmallIntegerField( |
881
|
|
|
choices=[x.value for x in STATUSES], |
882
|
|
|
help_text='example: Waiting', |
883
|
|
|
default=WAITING) |
884
|
|
|
|
885
|
|
|
# a field to track errors in UID loading. Should be blank if no errors |
886
|
|
|
# are found |
887
|
|
|
message = models.TextField( |
888
|
|
|
null=True, |
889
|
|
|
blank=True) |
890
|
|
|
|
891
|
|
|
# track biosample submission id in a field |
892
|
|
|
# HINT: if I update a completed submision, shuold I track the |
893
|
|
|
# last submission id? |
894
|
|
|
biosample_submission_id = models.CharField( |
895
|
|
|
max_length=255, |
896
|
|
|
blank=True, |
897
|
|
|
null=True, |
898
|
|
|
db_index=True, |
899
|
|
|
unique=True, |
900
|
|
|
help_text='Biosample submission id') |
901
|
|
|
|
902
|
|
|
owner = models.ForeignKey( |
903
|
|
|
User, |
904
|
|
|
related_name='submissions', |
905
|
|
|
on_delete=models.CASCADE) |
906
|
|
|
|
907
|
|
|
class Meta: |
908
|
|
|
# HINT: can I put two files for my cryoweb instance? May they have two |
909
|
|
|
# different version |
910
|
|
|
unique_together = (( |
911
|
|
|
"gene_bank_name", |
912
|
|
|
"gene_bank_country", |
913
|
|
|
"datasource_type", |
914
|
|
|
"datasource_version", |
915
|
|
|
"owner"),) |
916
|
|
|
|
917
|
|
|
def __str__(self): |
918
|
|
|
return "%s (%s, %s)" % ( |
919
|
|
|
self.gene_bank_name, |
920
|
|
|
self.gene_bank_country.label, |
921
|
|
|
self.datasource_version |
922
|
|
|
) |
923
|
|
|
|
924
|
|
|
def get_uploaded_file_basename(self): |
925
|
|
|
return os.path.basename(self.uploaded_file.name) |
926
|
|
|
|
927
|
|
|
def get_uploaded_file_path(self): |
928
|
|
|
"""Return uploaded file path in docker container""" |
929
|
|
|
|
930
|
|
|
# this is the full path in docker container |
931
|
|
|
fullpath = self.uploaded_file.file |
932
|
|
|
|
933
|
|
|
# get a string and quote fullpath |
934
|
|
|
return shlex.quote(str(fullpath)) |
935
|
|
|
|
936
|
|
|
def get_absolute_url(self): |
937
|
|
|
return reverse("submissions:detail", kwargs={"pk": self.pk}) |
938
|
|
|
|
939
|
|
|
def __can_I(self, names): |
940
|
|
|
"""Return True id self.status in statuses""" |
941
|
|
|
|
942
|
|
|
statuses = [x.value[0] for x in STATUSES if x.name in names] |
943
|
|
|
|
944
|
|
|
if self.status not in statuses: |
945
|
|
|
return True |
946
|
|
|
|
947
|
|
|
else: |
948
|
|
|
return False |
949
|
|
|
|
950
|
|
|
def can_edit(self): |
951
|
|
|
"""Returns True if I can edit a submission""" |
952
|
|
|
|
953
|
|
|
names = ['waiting', 'submitted'] |
954
|
|
|
|
955
|
|
|
return self.__can_I(names) |
956
|
|
|
|
957
|
|
|
def can_validate(self): |
958
|
|
|
names = ['error', 'waiting', 'submitted', 'completed'] |
959
|
|
|
|
960
|
|
|
return self.__can_I(names) |
961
|
|
|
|
962
|
|
|
def can_submit(self): |
963
|
|
|
names = ['ready'] |
964
|
|
|
|
965
|
|
|
# this is the opposite of self.__can_I |
966
|
|
|
statuses = [x.value[0] for x in STATUSES if x.name in names] |
967
|
|
|
|
968
|
|
|
# self.status need to be in statuses for submitting |
969
|
|
|
if self.status in statuses: |
970
|
|
|
return True |
971
|
|
|
|
972
|
|
|
else: |
973
|
|
|
return False |
974
|
|
|
|
975
|
|
|
|
976
|
|
|
# --- Custom functions |
977
|
|
|
|
978
|
|
|
|
979
|
|
|
# https://simpleisbetterthancomplex.com/tutorial/2016/07/22/how-to-extend-django-user-model.html#onetoone |
980
|
|
|
# we will now define signals so our Person model will be automatically |
981
|
|
|
# created/updated when we create/update User instances. |
982
|
|
|
# Basically we are hooking the create_user_person and save_user_person |
983
|
|
|
# methods to the User model, whenever a save event occurs. This kind of signal |
984
|
|
|
# is called post_save. |
985
|
|
|
# TODO: add default values when creating a superuser |
986
|
|
|
@receiver(post_save, sender=User) |
987
|
|
|
def create_user_person(sender, instance, created, **kwargs): |
988
|
|
|
if created: |
989
|
|
|
Person.objects.create(user=instance) |
990
|
|
|
|
991
|
|
|
|
992
|
|
|
@receiver(post_save, sender=User) |
993
|
|
|
def save_user_person(sender, instance, **kwargs): |
994
|
|
|
instance.person.save() |
995
|
|
|
|
996
|
|
|
|
997
|
|
|
# A method to truncate database |
998
|
|
|
def truncate_database(): |
999
|
|
|
"""Truncate image database""" |
1000
|
|
|
|
1001
|
|
|
logger.warning("Truncating ALL image tables") |
1002
|
|
|
|
1003
|
|
|
# call each class and truncate its table by calling truncate method |
1004
|
|
|
Animal.truncate() |
1005
|
|
|
DictBreed.truncate() |
1006
|
|
|
DictCountry.truncate() |
1007
|
|
|
DictRole.truncate() |
1008
|
|
|
DictSex.truncate() |
1009
|
|
|
DictSpecie.truncate() |
1010
|
|
|
DictUberon.truncate() |
1011
|
|
|
DictStage.truncate() |
1012
|
|
|
Name.truncate() |
1013
|
|
|
Ontology.truncate() |
1014
|
|
|
Organization.truncate() |
1015
|
|
|
Person.truncate() |
1016
|
|
|
Publication.truncate() |
1017
|
|
|
Sample.truncate() |
1018
|
|
|
Submission.truncate() |
1019
|
|
|
|
1020
|
|
|
logger.warning("All cryoweb tables were truncated") |
1021
|
|
|
|
1022
|
|
|
|
1023
|
|
|
def truncate_filled_tables(): |
1024
|
|
|
"""Truncate filled tables by import processes""" |
1025
|
|
|
|
1026
|
|
|
logger.warning("Truncating filled tables tables") |
1027
|
|
|
|
1028
|
|
|
# call each class and truncate its table by calling truncate method |
1029
|
|
|
Animal.truncate() |
1030
|
|
|
Name.truncate() |
1031
|
|
|
Publication.truncate() |
1032
|
|
|
Sample.truncate() |
1033
|
|
|
Submission.truncate() |
1034
|
|
|
|
1035
|
|
|
logger.warning("All filled tables were truncated") |
1036
|
|
|
|
1037
|
|
|
|
1038
|
|
|
def uid_report(user): |
1039
|
|
|
"""Performs a statistic on UID database to find issues. require user as |
1040
|
|
|
request.user""" |
1041
|
|
|
|
1042
|
|
|
report = {} |
1043
|
|
|
|
1044
|
|
|
# get n_of_animals |
1045
|
|
|
report['n_of_animals'] = Animal.objects.filter( |
1046
|
|
|
owner=user).count() |
1047
|
|
|
|
1048
|
|
|
# get n_of_samples |
1049
|
|
|
report['n_of_samples'] = Sample.objects.filter( |
1050
|
|
|
owner=user).count() |
1051
|
|
|
|
1052
|
|
|
# HINT: have they sense in a per user statistic? |
1053
|
|
|
|
1054
|
|
|
# check breeds without ontologies |
1055
|
|
|
breed = DictBreed.objects.filter(mapped_breed_term=None) |
1056
|
|
|
report['breeds_without_ontology'] = breed.count() |
1057
|
|
|
|
1058
|
|
|
# check countries without ontology |
1059
|
|
|
country = DictCountry.objects.filter(term=None) |
1060
|
|
|
report['countries_without_ontology'] = country.count() |
1061
|
|
|
|
1062
|
|
|
# check species without ontology |
1063
|
|
|
species = DictSpecie.objects.filter(term=None) |
1064
|
|
|
report['species_without_ontology'] = species.count() |
1065
|
|
|
|
1066
|
|
|
return report |
1067
|
|
|
|
1068
|
|
|
|
1069
|
|
|
# A method to discover is image database has data or not |
1070
|
|
|
def db_has_data(): |
1071
|
|
|
# Test only tables I read data to fill UID |
1072
|
|
|
if (Animal.objects.exists() or Sample.objects.exists() or |
1073
|
|
|
Name.objects.exists()): |
1074
|
|
|
return True |
1075
|
|
|
|
1076
|
|
|
else: |
1077
|
|
|
return False |
1078
|
|
|
|