Completed
Push — master ( 79309a...685571 )
by Andrea
01:12
created

Federation._remove_deleted_entities()   A

Complexity

Conditions 3

Size

Total Lines 9

Duplication

Lines 0
Ratio 0 %

Importance

Changes 2
Bugs 0 Features 0
Metric Value
dl 0
loc 9
rs 9.6666
c 2
b 0
f 0
cc 3
1
#################################################################
2
# MET v2 Metadate Explorer Tool
3
#
4
# This Software is Open Source. See License: https://github.com/TERENA/met/blob/master/LICENSE.md
5
# Copyright (c) 2012, TERENA All rights reserved.
6
#
7
# This Software is based on MET v1 developed for TERENA by Yaco Sistemas, http://www.yaco.es/
8
# MET v2 was developed for TERENA by Tamim Ziai, DAASI International GmbH, http://www.daasi.de
9
# Current version of MET has been revised for performance improvements by Andrea Biancini,
10
# Consortium GARR, http://www.garr.it
11
#########################################################################################
12
13
import pytz
14
import simplejson as json
15
16
from datetime import datetime, time, timedelta
17
18
from django.contrib import messages
19
from django.core.urlresolvers import reverse
20
from django.db import models
21
from django.db.models import Max
22
from django.db.models.signals import pre_save
23
from django.utils.safestring import mark_safe
24
from django.utils.translation import ugettext_lazy as _
25
from django.utils import timezone
26
from django.dispatch import receiver
27
from django.template.defaultfilters import slugify
28
29
from met.metadataparser.xmlparser import MetadataParser
30
31
from base import Base
32
from entity import Entity
33
from entity_type import EntityType
34
from entity_stat import EntityStat, stats
35
from entity_federations import Entity_Federations
36
37
FEDERATION_TYPES = (
38
    (None, ''),
39
    ('hub-and-spoke', 'Hub and Spoke'),
40
    ('mesh', 'Full Mesh'),
41
)
42
43
def update_obj(mobj, obj, attrs=None):
44
    for_attrs = attrs or getattr(mobj, 'all_attrs', [])
45
    for attrb in attrs or for_attrs:
46
        if (getattr(mobj, attrb, None) and
47
            getattr(obj, attrb, None) and
48
            getattr(mobj, attrb) != getattr(obj, attrb)):
49
            setattr(obj, attrb, getattr(mobj, attrb))
50
51
class Federation(Base):
52
    name = models.CharField(blank=False, null=False, max_length=200,
53
                            unique=True, verbose_name=_(u'Name'))
54
55
    type = models.CharField(blank=True, null=True, max_length=100,
56
                            unique=False, verbose_name=_(u'Type'), choices=FEDERATION_TYPES)
57
58
    url = models.URLField(verbose_name='Federation url',
59
                          blank=True, null=True)
60
    
61
    fee_schedule_url = models.URLField(verbose_name='Fee schedule url',
62
                                       max_length=150, blank=True, null=True)
63
64
    logo = models.ImageField(upload_to='federation_logo', blank=True,
65
                             null=True, verbose_name=_(u'Federation logo'))
66
67
    is_interfederation = models.BooleanField(default=False, db_index=True,
68
                                         verbose_name=_(u'Is interfederation'))
69
70
    slug = models.SlugField(max_length=200, unique=True)
71
72
    country = models.CharField(blank=True, null=True, max_length=100,
73
                               unique=False, verbose_name=_(u'Country'))
74
75
    metadata_update = models.DateField(blank=True, null=True,
76
                                       unique=False, verbose_name=_(u'Metadata update date'))
77
78
    certstats = models.CharField(blank=True, null=True, max_length=200,
79
                                 unique=False, verbose_name=_(u'Certificate Stats'))
80
81
    @property
82
    def certificates(self):
83
        return json.loads(self.certstats)
84
85
    @property
86
    def _metadata(self):
87
        if not hasattr(self, '_metadata_cache'):
88
            self._metadata_cache = self.load_file()
89
        return self._metadata_cache
90
    def __unicode__(self):
91
        return self.name
92
93
    def get_entity_metadata(self, entityid):
94
        return self._metadata.get_entity(entityid)
95
96
    def get_entity(self, entityid):
97
        return self.entity_set.get(entityid=entityid)
98
99
    def process_metadata(self):
100
        metadata = self.load_file()
101
102
        if self.file_id and metadata.file_id and metadata.file_id == self.file_id:
103
            return
104
        else:
105
            self.file_id = metadata.file_id
106
107
        if not metadata:
108
            return
109
        if not metadata.is_federation:
110
            raise XmlDescriptionError("XML Haven't federation form")
111
112
        update_obj(metadata.get_federation(), self)
113
        self.certstats = MetadataParser.get_certstats(metadata.rootelem)
114
115
    def _remove_deleted_entities(self, entities_from_xml, request):
116
        removed = 0
117
        for entity in self.entity_set.all():
118
            #Remove entity relation if does not exist in metadata
119
            if not entity.entityid in entities_from_xml:
120
                Entity_Federations.objects.filter(federation=self, entity=entity).delete()
121
                removed += 1
122
123
        return removed
124
125
    def _update_entities(self, entities_to_update, entities_to_add):
126
        for e in entities_to_update:
127
            e.save()
128
129
        for e in entities_to_add:
130
            membership = Entity_Federations.objects.get_or_create(federation=self, entity=e)[0]
131
            membership.registration_instant = e.registration_instant.date() if e.registration_instant else None
132
            membership.save()
133
134
    def _add_new_entities(self, entities, entities_from_xml, request, federation_slug):
135
        db_entity_types = EntityType.objects.all()
136
        cached_entity_types = { entity_type.xmlname: entity_type for entity_type in db_entity_types }
137
138
        entities_to_add = []
139
        entities_to_update = []
140
141
        for m_id in entities_from_xml:
142
            if request and federation_slug:
143
                request.session['%s_cur_entities' % federation_slug] += 1
144
                request.session.save()
145
146
            created = False
147
            if m_id in entities:
148
                entity = entities[m_id]
149
            else:
150
                entity, created = Entity.objects.get_or_create(entityid=m_id)
151
152
            entityid = entity.entityid
153
            name = entity.name
154
            registration_authority = entity.registration_authority
155
            certstats = entity.certstats
156
            display_protocols = entity._display_protocols
157
 
158
            entity_from_xml = self._metadata.get_entity(m_id, False)
159
            entity.process_metadata(False, entity_from_xml, cached_entity_types)
160
161
            if created or entity.has_changed(entityid, name, registration_authority, certstats, display_protocols):
162
                entities_to_update.append(entity)
163
164
            entities_to_add.append(entity)
165
166
        self._update_entities(entities_to_update, entities_to_add)
167
        return len(entities_to_update) 
168
169
    @staticmethod
170
    def _daterange(start_date, end_date):
171
        for n in range(int ((end_date - start_date).days + 1)):
172
            yield start_date + timedelta(n)
173
174
    def compute_new_stats(self):
175
        entities_from_xml = self._metadata.get_entities()
176
177
        entities = Entity.objects.filter(entityid__in=entities_from_xml)
178
        entities = entities.prefetch_related('types')
179
        memberships = Entity_Federations.objects.filter(federation=self)
180
181
        try:
182
            first_date = EntityStat.objects.filter(federation=self).aggregate(Max('time'))['time__max']
183
            if not first_date:
184
                raise Exception('Not able to find statistical data in the DB.')
185
        except Exception:
186
            first_date = datetime(2010, 1, 1)
187
            first_date = pytz.utc.localize(first_date)
188
      
189
        for curtimestamp in self._daterange(first_date, timezone.now()):
190
            computed = {}
191
            not_computed = []
192
            entity_stats = []
193
            for feature in stats['features'].keys():
194
                fun = getattr(self, 'get_%s' % feature, None)
195
    
196
                if callable(fun):
197
                    stat = EntityStat()
198
                    stat.feature = feature
199
                    stat.time = curtimestamp
200
                    stat.federation = self
201
                    stat.value = fun(entities, stats['features'][feature], curtimestamp)
202
                    entity_stats.append(stat)
203
                    computed[feature] = stat.value
204
                else:
205
                    not_computed.append(feature)
206
207
            from_time = datetime.combine(curtimestamp, time.min) 
208
            if timezone.is_naive(from_time):
209
                from_time = pytz.utc.localize(from_time)
210
            to_time = datetime.combine(curtimestamp, time.max)
211
            if timezone.is_naive(to_time):
212
                to_time = pytz.utc.localize(to_time)
213
214
            EntityStat.objects.filter(federation=self, time__gte=from_time, time__lte=to_time).delete()
215
            EntityStat.objects.bulk_create(entity_stats)
216
217
        return (computed, not_computed)
218
219
    def process_metadata_entities(self, request=None, federation_slug=None):
220
        entities_from_xml = self._metadata.get_entities()
221
        removed = self._remove_deleted_entities(entities_from_xml, request)
222
223
        entities = {}
224
        db_entities = Entity.objects.filter(entityid__in=entities_from_xml)
225
        db_entities = db_entities.prefetch_related('types', 'entity_categories')
226
227
        for entity in db_entities.all():
228
            entities[entity.entityid] = entity
229
230
        if request and federation_slug:
231
            request.session['%s_num_entities' % federation_slug] = len(entities_from_xml)
232
            request.session['%s_cur_entities' % federation_slug] = 0
233
            request.session['%s_process_done' % federation_slug] = False
234
            request.session.save()
235
236
        updated = self._add_new_entities(entities, entities_from_xml, request, federation_slug)
237
238
        if request and federation_slug:
239
            request.session['%s_process_done' % federation_slug] = True
240
            request.session.save()
241
242
        return removed, updated
243
244
    def get_absolute_url(self):
245
        return reverse('federation_view', args=[self.slug])
246
247
    @classmethod
248
    def get_sp(cls, entities, xml_name, ref_date=None):
249
        if ref_date and ref_date < pytz.utc.localize(datetime.now() - timedelta(days = 1)):
250
            selected = entities.filter(types__xmlname=xml_name, entity_federations__registration_instant__lt = ref_date)
251
        else:
252
            selected = entities.filter(types__xmlname=xml_name)
253
        return len(selected)
254
255
    @classmethod
256
    def get_idp(cls, entities, xml_name, ref_date=None):
257
        if ref_date and ref_date < pytz.utc.localize(datetime.now() - timedelta(days = 1)):
258
            selected = entities.filter(types__xmlname=xml_name, entity_federations__registration_instant__lt = ref_date)
259
        else:
260
            selected = entities.filter(types__xmlname=xml_name)
261
        return len(selected)
262
263
    @classmethod
264
    def get_aa(cls, entities, xml_name, ref_date=None):
265
        if ref_date and ref_date < pytz.utc.localize(datetime.now() - timedelta(days = 1)):
266
            selected = entities.filter(types__xmlname=xml_name, entity_federations__registration_instant__lt = ref_date)
267
        else:
268
            selected = entities.filter(types__xmlname=xml_name)
269
        return len(selected)
270
271
    def get_sp_saml1(self, entities, xml_name, ref_date = None):
272
        return self.get_stat_protocol(entities, xml_name, 'SPSSODescriptor', ref_date)
273
274
    def get_sp_saml2(self, entities, xml_name, ref_date = None):
275
        return self.get_stat_protocol(entities, xml_name, 'SPSSODescriptor', ref_date)
276
277
    def get_sp_shib1(self, entities, xml_name, ref_date = None):
278
        return self.get_stat_protocol(entities, xml_name, 'SPSSODescriptor', ref_date)
279
280
    def get_idp_saml1(self, entities, xml_name, ref_date = None):
281
        return self.get_stat_protocol(entities, xml_name, 'IDPSSODescriptor', ref_date)
282
283
    def get_idp_saml2(self, entities, xml_name, ref_date = None):
284
        return self.get_stat_protocol(entities, xml_name, 'IDPSSODescriptor', ref_date)
285
286
    def get_idp_shib1(self, entities, xml_name, ref_date = None):
287
        return self.get_stat_protocol(entities, xml_name, 'IDPSSODescriptor', ref_date)
288
289
    def get_stat_protocol(self, entities, xml_name, service_type, ref_date):
290
        if ref_date and ref_date < pytz.utc.localize(datetime.now() - timedelta(days = 1)):
291
            selected = entities.filter(types__xmlname=service_type, _display_protocols__contains=xml_name, entity_federations__registration_instant__lt = ref_date)
292
        else:
293
            selected = entities.filter(types__xmlname=service_type, _display_protocols__contains=xml_name)
294
        return len(selected)
295
296
    def can_edit(self, user, delete):
297
        if user.is_superuser:
298
            return True
299
300
        permission = 'delete_federation' if delete else 'change_federation'
301
        if user.has_perm('metadataparser.%s' % permission) and user in self.editor_users.all():
302
            return True
303
        return False
304
305
@receiver(pre_save, sender=Federation, dispatch_uid='federation_pre_save')
306
def federation_pre_save(sender, instance, **kwargs):
307
    # Skip pre_save if only file name is saved
308
    if kwargs.has_key('update_fields') and kwargs['update_fields'] == set(['file']):
309
        return
310
311
    #slug = slugify(unicode(instance.name))[:200]
312
    #if instance.file_url and instance.file_url != '':
313
    #    try:
314
    #        instance.fetch_metadata_file(slug)
315
    #    except Exception, e:
316
    #        pass
317
318
    if instance.name:
319
        instance.slug = slugify(unicode(instance))[:200]
320
321
322
@receiver(pre_save, sender=Entity, dispatch_uid='entity_pre_save')
323
def entity_pre_save(sender, instance, **kwargs):
324
    #if refetch and instance.file_url:
325
    #    slug = slugify(unicode(instance.name))[:200]
326
    #    instance.fetch_metadata_file(slug)
327
    #    instance.process_metadata()
328
    pass
329