common.tasks.redis_lock() - Code Metrics - Inspection of ":zap: Batch submission to Biosamples" - cnr-ibba/IMAGE-InjectTool - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Pull Request — master (#44)

by Paolo

created 2019-09-23 12:31 UTC

common.tasks.redis_lock() B

↳ Parent: common.tasks

Complexity

Conditions

Size

Total Lines	52
Code Lines	18

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
eloc	18
dl	0
loc	52
rs	8.5666
c	0
b	0
f	0
cc	6
nop	3

How to fix Long Method

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Jan 15 16:42:24 2019

@author: Paolo Cozzi <[email protected]>
"""

import redis
import logging

from contextlib import contextmanager
from celery.five import monotonic

from django.conf import settings

from submissions.helpers import send_message
from validation.helpers import construct_validation_message
from common.constants import NEED_REVISION, ERROR

# Lock expires in 10 minutes
LOCK_EXPIRE = 60 * 10

# Get an instance of a logger
logger = logging.getLogger(__name__)


class BatchFailurelMixin():
    """Common mixin for batch task failure. Need to setup ``batch_type``
    (update/delete) and ``model_type`` (animal/sample)
    """

    batch_type = None
    model_type = None
    submission_cls = None

    # Ovverride default on failure method
    # This is not a failed validation for a wrong value, this is an
    # error in task that mean an error in coding
    def on_failure(self, exc, task_id, args, kwargs, einfo):
        logger.error('{0!r} failed: {1!r}'.format(task_id, exc))

        submission_id = args[0]

        logger.error(
            ("%s called with %s" % (self.name, args))
        )

        # get submission object
        submission_obj = self.submission_cls.objects.get(pk=submission_id)

        # mark submission with ERROR
        submission_obj.status = ERROR
        submission_obj.message = (
            "Error in %s batch %s: %s" % (
                self.model_type, self.batch_type, str(exc)))
        submission_obj.save()

        send_message(submission_obj)

        # send a mail to the user with the stacktrace (einfo)
        submission_obj.owner.email_user(
            "Error in %s batch %s for submission: %s" % (
                self.model_type, self.batch_type, submission_obj.id),
            ("Something goes wrong in batch %s for %ss. Please report "
             "this to InjectTool team\n\n %s" % (
                self.model_type, self.batch_type, str(einfo))),
        )

        # TODO: submit mail to admin


class BatchUpdateMixin:
    """Mixin to do batch update of fields to fix validation"""

    item_cls = None
    submission_cls = None

    def batch_update(self, submission_id, ids, attribute):
        for id_, value in ids.items():
            if value == '' or value == 'None':
                value = None

            item_object = self.item_cls.objects.get(pk=id_)

            if getattr(item_object, attribute) != value:
                setattr(item_object, attribute, value)
                item_object.save()

        # Update submission
        submission_obj = self.submission_cls.objects.get(pk=submission_id)
        submission_obj.status = NEED_REVISION
        submission_obj.message = "Data updated, try to rerun validation"
        submission_obj.save()

        send_message(
            submission_obj, construct_validation_message(submission_obj)
        )


@contextmanager
def redis_lock(lock_id, blocking=False, expire=True):
    """
    This function get a lock relying on a lock name and other status. You
    can describe more process using the same lock name and give exclusive
    access to one of them.

    Args:
        lock_id (str): the name of the lock to take
        blocking (bool): if True, we wait until we have the block, if False
            we returns immediately False
        expire (bool): if True, lock will expire after LOCK_EXPIRE timeout,
            if False, it will persist until lock is released

    Returns:
        bool: True if lock acquired, False otherwise
    """

    # read parameters from settings
    REDIS_CLIENT = redis.StrictRedis(
        host=settings.REDIS_HOST,
        port=settings.REDIS_PORT,
        db=settings.REDIS_DB)

    # this will be the redis lock
    lock = None

    # timeout for the lock (if expire condition)
    timeout_at = monotonic() + LOCK_EXPIRE - 3

    if expire:
        lock = REDIS_CLIENT.lock(lock_id, timeout=LOCK_EXPIRE)

    else:
        lock = REDIS_CLIENT.lock(lock_id, timeout=None)

    status = lock.acquire(blocking=blocking)

    try:
        logger.debug("lock %s acquired is: %s" % (lock_id, status))
        yield status

    finally:
        # we take advantage of using add() for atomic locking
        # don't release the lock if we didn't acquire it
        if status and ((monotonic() < timeout_at and expire) or not expire):
            logger.debug("Releasing lock %s" % lock_id)
            # don't release the lock if we exceeded the timeout
            # to lessen the chance of releasing an expired lock
            # owned by someone else
            # if no timeout and lock is taken, release it
            lock.release()


1			#!/usr/bin/env python3
2			# -- coding: utf-8 --
3			"""
4			Created on Tue Jan 15 16:42:24 2019
5
6			@author: Paolo Cozzi <[email protected]>
7			"""
8
9			import redis
10			import logging
11
12			from contextlib import contextmanager
13			from celery.five import monotonic
14
15			from django.conf import settings
16
17			from submissions.helpers import send_message
18			from validation.helpers import construct_validation_message
19			from common.constants import NEED_REVISION, ERROR
20
21			# Lock expires in 10 minutes
22			LOCK_EXPIRE = 60 * 10
23
24			# Get an instance of a logger
25			logger = logging.getLogger(__name__)
26
27
28			class BatchFailurelMixin():
29			"""Common mixin for batch task failure. Need to setup ``batch_type``
30			(update/delete) and ``model_type`` (animal/sample)
31			"""
32
33			batch_type = None
34			model_type = None
35			submission_cls = None
36
37			# Ovverride default on failure method
38			# This is not a failed validation for a wrong value, this is an
39			# error in task that mean an error in coding
40			def on_failure(self, exc, task_id, args, kwargs, einfo):
41			logger.error('{0!r} failed: {1!r}'.format(task_id, exc))
42
43			submission_id = args[0]
44
45			logger.error(
46			("%s called with %s" % (self.name, args))
47			)
48
49			# get submission object
50			submission_obj = self.submission_cls.objects.get(pk=submission_id)
51
52			# mark submission with ERROR
53			submission_obj.status = ERROR
54			submission_obj.message = (
55			"Error in %s batch %s: %s" % (
56			self.model_type, self.batch_type, str(exc)))
57			submission_obj.save()
58
59			send_message(submission_obj)
60
61			# send a mail to the user with the stacktrace (einfo)
62			submission_obj.owner.email_user(
63			"Error in %s batch %s for submission: %s" % (
64			self.model_type, self.batch_type, submission_obj.id),
65			("Something goes wrong in batch %s for %ss. Please report "
66			"this to InjectTool team\n\n %s" % (
67			self.model_type, self.batch_type, str(einfo))),
68			)
69
70			# TODO: submit mail to admin
71
72
73			class BatchUpdateMixin:
74			"""Mixin to do batch update of fields to fix validation"""
75
76			item_cls = None
77			submission_cls = None
78
79			def batch_update(self, submission_id, ids, attribute):
80			for id_, value in ids.items():
81			if value == '' or value == 'None':
82			value = None
83
84			item_object = self.item_cls.objects.get(pk=id_)
85
86			if getattr(item_object, attribute) != value:
87			setattr(item_object, attribute, value)
88			item_object.save()
89
90			# Update submission
91			submission_obj = self.submission_cls.objects.get(pk=submission_id)
92			submission_obj.status = NEED_REVISION
93			submission_obj.message = "Data updated, try to rerun validation"
94			submission_obj.save()
95
96			send_message(
97			submission_obj, construct_validation_message(submission_obj)
98			)
99
100
101			@contextmanager
102			def redis_lock(lock_id, blocking=False, expire=True):
103			"""
104			This function get a lock relying on a lock name and other status. You
105			can describe more process using the same lock name and give exclusive
106			access to one of them.
107
108			Args:
109			lock_id (str): the name of the lock to take
110			blocking (bool): if True, we wait until we have the block, if False
111			we returns immediately False
112			expire (bool): if True, lock will expire after LOCK_EXPIRE timeout,
113			if False, it will persist until lock is released
114
115			Returns:
116			bool: True if lock acquired, False otherwise
117			"""
118
119			# read parameters from settings
120			REDIS_CLIENT = redis.StrictRedis(
121			host=settings.REDIS_HOST,
122			port=settings.REDIS_PORT,
123			db=settings.REDIS_DB)
124
125			# this will be the redis lock
126			lock = None
127
128			# timeout for the lock (if expire condition)
129			timeout_at = monotonic() + LOCK_EXPIRE - 3
130
131			if expire:
132			lock = REDIS_CLIENT.lock(lock_id, timeout=LOCK_EXPIRE)
133
134			else:
135			lock = REDIS_CLIENT.lock(lock_id, timeout=None)
136
137			status = lock.acquire(blocking=blocking)
138
139			try:
140			logger.debug("lock %s acquired is: %s" % (lock_id, status))
141			yield status
142
143			finally:
144			# we take advantage of using add() for atomic locking
145			# don't release the lock if we didn't acquire it
146			if status and ((monotonic() < timeout_at and expire) or not expire):
147			logger.debug("Releasing lock %s" % lock_id)
148			# don't release the lock if we exceeded the timeout
149			# to lessen the chance of releasing an expired lock
150			# owned by someone else
151			# if no timeout and lock is taken, release it
152			lock.release()
153

cnr-ibba / IMAGE-InjectTool

Pull Request — master (#44)

common.tasks.redis_lock() B

Complexity

Size

Duplication

Importance

How to fix Long Method

Long Method

Duplication Side-by-Side

Filter issues like