submissions.forms

Complexity

Total Complexity

24

Size/Duplication

Total Lines	263
Duplicated Lines	0 %

Importance

Changes

0

6 Methods

Rating	Name	Duplication	Size	Complexity
A	UniqueSubmissionMixin.check_submission_exists()	0	35	5
B	SubmissionFormMixin.clean()	0	21	8
A	UpdateSubmissionForm.clean()	0	3	1
A	SubmissionFormMixin.check_crbanim_columns()	0	19	2
A	SubmissionFormMixin.check_file_encoding()	0	24	2
B	SubmissionFormMixin.check_template_file()	0	40	6

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Jul 24 15:51:05 2018

@author: Paolo Cozzi <cozzi@ibba.cnr.it>
"""

import magic
import logging
import tempfile

from django import forms
from django.conf import settings

from common.constants import CRB_ANIM_TYPE, TEMPLATE_TYPE
from common.forms import RequestFormMixin
from common.helpers import get_admin_emails
from uid.models import Submission
from crbanim.helpers import CRBAnimReader
from excel.helpers import ExcelTemplateReader

# Get an instance of a logger
logger = logging.getLogger(__name__)


class UniqueSubmissionMixin():
    # a custom attribute in order to determine if I'm reloading or not
    is_reload = False

    def check_submission_exists(self):
        """Test if I already have a submission with the same data"""

        logger.info(self.changed_data)

        if (self.is_reload and (
                'datasource_type' not in self.changed_data and
                'datasource_version' not in self.changed_data)):

            logger.info("Replacing a datasource: %s" % self.cleaned_data)

            # I'm replacing data, no need to test if a submission with the
            # same data exists
            return

        # get unique attributes
        unique_together = Submission._meta.unique_together[0]

        # get submitted attributes
        data = {key: self.cleaned_data.get(key) for key in unique_together
                if self.cleaned_data.get(key) is not None}

        # ovverride owner attribute
        data['owner'] = self.request.user

        # test for a submission object with the same attributes
        if Submission.objects.filter(**data).exists():
            msg = (
                "Error: There is already a submission with the same "
                "attributes. Please change one of the following: "
                "Gene bank name, Gene bank country, Data source type and "
                "Data source version")

            # raising an exception:
            raise forms.ValidationError(msg, code='invalid')


class SubmissionFormMixin(UniqueSubmissionMixin):
    def clean(self):
        # test if I have a submission with the provided data
        self.check_submission_exists()

        # I can call this method without providing a 'uploaded file'
        # (for instance, when omitting uploaded file)
        if "uploaded_file" in self.cleaned_data:
            # avoid file type for excel types (is not an text file)
            if ("datasource_type" in self.cleaned_data and
                    self.cleaned_data["datasource_type"] != TEMPLATE_TYPE):
                self.check_file_encoding()

            # check crbanim files only if provided
            if ("datasource_type" in self.cleaned_data and
                    self.cleaned_data["datasource_type"] == CRB_ANIM_TYPE):
                self.check_crbanim_columns()

            # check template files only if provided
            if ("datasource_type" in self.cleaned_data and
                    self.cleaned_data["datasource_type"] == TEMPLATE_TYPE):
                self.check_template_file()

    def check_file_encoding(self):
        uploaded_file = self.cleaned_data['uploaded_file']

        # read one chunk of such file
        chunk = next(uploaded_file.chunks())
        magic_line = magic.from_buffer(chunk)
        file_type = magic_line.split(",")[0]

        # changed cause the different behavior of libmagic1
        if file_type not in [
                "CSV text",
                "ASCII",
                "UTF-8",
                "UTF-8 Unicode text"
                ]:

            # create message and add error
            msg = (
                "Error: file is not a CSV in UTF-8 nor ASCII format: "
                "format was '%s'" % file_type)
            logger.error(msg)

            # raising an exception:
            raise forms.ValidationError(msg, code='invalid')

    def check_crbanim_columns(self):
        """Check if a CRBanim file has mandatory columns"""

        uploaded_file = self.cleaned_data['uploaded_file']

        # read one chunk of such file
        chunk = next(uploaded_file.chunks())

        # now determine if CRBanim file is valid. chunk is in binary format
        # neet to convert to a string, fortunately I've already check that
        # file is in UTF-8
        check, not_found = CRBAnimReader.is_valid(chunk.decode("utf-8"))

        if check is False:
            msg = "Error: file lacks of CRBanim mandatory columns: %s" % (
                not_found)

            # raising an exception:
            raise forms.ValidationError(msg, code='invalid')

    def check_template_file(self):
        """Check if template file has columns and sheets"""

        uploaded_file = self.cleaned_data['uploaded_file']

        chunk = next(uploaded_file.chunks())
        magic_line = magic.from_buffer(chunk)

        if 'Microsoft' not in magic_line:
            msg = "The file you provided is not a Template file"
            raise forms.ValidationError(msg, code='invalid')

        # xlrd can manage only files. Write a temporary file
        with tempfile.NamedTemporaryFile(delete=True) as tmpfile:
            for chunk in uploaded_file.chunks():
                tmpfile.write(chunk)

            # open the file with proper model
            reader = ExcelTemplateReader()
            reader.read_file(tmpfile.name)

            # check that template has at least breed, animal, sample sheets
            check, not_found = reader.check_sheets()

            if check is False:
                msg = "Error: file lacks of Template mandatory sheets: %s" % (
                    not_found)

                # raising an exception:
                raise forms.ValidationError(msg, code='invalid')

            # check that template has at least breed, animal, sample sheets
            check, not_found = reader.check_columns()

            if check is False:
                msg = "Error: file lacks of Template mandatory columns: %s" % (
                    not_found)

                # raising an exception:
                raise forms.ValidationError(msg, code='invalid')


class SubmissionForm(SubmissionFormMixin, RequestFormMixin, forms.ModelForm):
    class Meta:
        model = Submission
        fields = (
            'title',
            'description',
            'gene_bank_name',
            'gene_bank_country',
            'organization',
            'datasource_type',
            'datasource_version',
            'uploaded_file'
        )

        help_texts = {
            'uploaded_file': 'Need to be in UTF-8 or ASCII format',
            'organization': (
                """Who owns the data. Not listed? please """
                """<a href="mailto:{0}?subject=please add my organization">"""
                """contact us</a>""".format(get_admin_emails()[0])
            ),
            'datasource_type': (
                """example: CryoWeb. Need an empty template file? """
                """download it from <a href="%s%s">here</a>""" % (
                    settings.MEDIA_URL,
                    "Image_sample_empty_template_20191002_v2.1.xlsx")
            )
        }


# I use forms.Form since I need to pass primary key as a field,
# and I can't use it with a modelform
class ReloadForm(SubmissionFormMixin, RequestFormMixin, forms.ModelForm):
    # a custom attribute in order to determine if I'm reloading or not
    is_reload = True

    # custom attributes
    agree_reload = forms.BooleanField(
        label="That's fine. Replace my submission data with this file",
        help_text="You have to check this box to reload your data")

    class Meta:
        model = Submission
        fields = (
            'datasource_type',
            'datasource_version',
            'uploaded_file',
        )

        help_texts = {
            'uploaded_file': 'Need to be in UTF-8 or ASCII format',
            'datasource_type': (
                """example: CryoWeb. Need an empty template file? """
                """download it from <a href="%s%s">here</a>""" % (
                    settings.MEDIA_URL,
                    "Image_sample_empty_template_20191002_v2.1.xlsx")
            )
        }


class UpdateSubmissionForm(
        UniqueSubmissionMixin, RequestFormMixin, forms.ModelForm):
    class Meta:
        model = Submission
        fields = (
            'title',
            'description',
            'gene_bank_name',
            'gene_bank_country',
            'organization',
            "datasource_type",
            "datasource_version",
        )

        help_texts = {
            'organization': (
                """Who owns the data. Not listed? please """
                """<a href="mailto:{0}?subject=please add my organization">"""
                """contact us</a>""".format(get_admin_emails()[0])
            )
        }

    def clean(self):
        # test if I have a submission with the provided data
        self.check_submission_exists()