Completed
Push — master ( 14bff8...d3739e )
by Andrea
01:16
created

Federation.process_metadata_entities()   C

Complexity

Conditions 7

Size

Total Lines 25

Duplication

Lines 0
Ratio 0 %

Importance

Changes 2
Bugs 0 Features 0
Metric Value
c 2
b 0
f 0
dl 0
loc 25
rs 5.5
cc 7
1
#################################################################
2
# MET v2 Metadate Explorer Tool
3
#
4
# This Software is Open Source. See License: https://github.com/TERENA/met/blob/master/LICENSE.md
5
# Copyright (c) 2012, TERENA All rights reserved.
6
#
7
# This Software is based on MET v1 developed for TERENA by Yaco Sistemas, http://www.yaco.es/
8
# MET v2 was developed for TERENA by Tamim Ziai, DAASI International GmbH, http://www.daasi.de
9
# Current version of MET has been revised for performance improvements by Andrea Biancini,
10
# Consortium GARR, http://www.garr.it
11
#########################################################################################
12
13
import pytz
14
import simplejson as json
15
16
from datetime import datetime, time, timedelta
17
18
from django.core.urlresolvers import reverse
19
from django.db import models
20
from django.db.models import Max
21
from django.db.models.signals import pre_save
22
from django.utils.translation import ugettext_lazy as _
23
from django.utils import timezone
24
from django.dispatch import receiver
25
from django.template.defaultfilters import slugify
26
27
from met.metadataparser.xmlparser import MetadataParser
28
29
from met.metadataparser.models.base import Base, XmlDescriptionError
30
from met.metadataparser.models.entity import Entity
31
from met.metadataparser.models.entity_type import EntityType
32
from met.metadataparser.models.entity_stat import EntityStat, stats
33
from met.metadataparser.models.entity_federations import Entity_Federations
34
35
FEDERATION_TYPES = (
36
    (None, ''),
37
    ('hub-and-spoke', 'Hub and Spoke'),
38
    ('mesh', 'Full Mesh'),
39
)
40
41
def update_obj(mobj, obj, attrs=None):
42
    for_attrs = attrs or getattr(mobj, 'all_attrs', [])
43
    for attrb in attrs or for_attrs:
44
        if (getattr(mobj, attrb, None) and
45
            getattr(obj, attrb, None) and
46
            getattr(mobj, attrb) != getattr(obj, attrb)):
47
            setattr(obj, attrb, getattr(mobj, attrb))
48
49
class Federation(Base):
50
    name = models.CharField(blank=False, null=False, max_length=200,
51
                            unique=True, verbose_name=_(u'Name'))
52
53
    type = models.CharField(blank=True, null=True, max_length=100,
54
                            unique=False, verbose_name=_(u'Type'), choices=FEDERATION_TYPES)
55
56
    url = models.URLField(verbose_name='Federation url',
57
                          blank=True, null=True)
58
    
59
    fee_schedule_url = models.URLField(verbose_name='Fee schedule url',
60
                                       max_length=150, blank=True, null=True)
61
62
    logo = models.ImageField(upload_to='federation_logo', blank=True,
63
                             null=True, verbose_name=_(u'Federation logo'))
64
65
    is_interfederation = models.BooleanField(default=False, db_index=True,
66
                                             verbose_name=_(u'Is interfederation'))
67
68
    slug = models.SlugField(max_length=200, unique=True)
69
70
    country = models.CharField(blank=True, null=True, max_length=100,
71
                               unique=False, verbose_name=_(u'Country'))
72
73
    metadata_update = models.DateField(blank=True, null=True,
74
                                       unique=False, verbose_name=_(u'Metadata update date'))
75
76
    certstats = models.CharField(blank=True, null=True, max_length=200,
77
                                 unique=False, verbose_name=_(u'Certificate Stats'))
78
79
    @property
80
    def certificates(self):
81
        return json.loads(self.certstats)
82
83
    @property
84
    def _metadata(self):
85
        if not hasattr(self, '_metadata_cache'):
86
            self._metadata_cache = self.load_file()
87
        return self._metadata_cache
88
    def __unicode__(self):
89
        return self.name
90
91
    def get_entity_metadata(self, entityid):
92
        return self._metadata.get_entity(entityid)
93
94
    def get_entity(self, entityid):
95
        return self.entity_set.get(entityid=entityid)
96
97
    def process_metadata(self):
98
        metadata = self.load_file()
99
100
        if self.file_id and metadata.file_id and metadata.file_id == self.file_id:
101
            return
102
        else:
103
            self.file_id = metadata.file_id
104
105
        if not metadata:
106
            return
107
        if not metadata.is_federation:
108
            raise XmlDescriptionError("XML Haven't federation form")
109
110
        update_obj(metadata.get_federation(), self)
111
        self.certstats = MetadataParser.get_certstats(metadata.rootelem)
112
113
    def _remove_deleted_entities(self, entities_from_xml):
114
        removed = 0
115
        for entity in self.entity_set.all():
116
            #Remove entity relation if does not exist in metadata
117
            if not entity.entityid in entities_from_xml:
118
                Entity_Federations.objects.filter(federation=self, entity=entity).delete()
119
                removed += 1
120
121
        return removed
122
123
    def _update_entities(self, entities_to_update, entities_to_add):
124
        for e in entities_to_update:
125
            e.save()
126
127
        for e in entities_to_add:
128
            membership = Entity_Federations.objects.get_or_create(federation=self, entity=e)[0]
129
            membership.registration_instant = e.registration_instant.date() if e.registration_instant else None
130
            membership.save()
131
132
    def _add_new_entities(self, entities, entities_from_xml, request, federation_slug):
133
        db_entity_types = EntityType.objects.all()
134
        cached_entity_types = { entity_type.xmlname: entity_type for entity_type in db_entity_types }
135
136
        entities_to_add = []
137
        entities_to_update = []
138
139
        for m_id in entities_from_xml:
140
            if request and federation_slug:
141
                request.session['%s_cur_entities' % federation_slug] += 1
142
                request.session.save()
143
144
            created = False
145
            if m_id in entities:
146
                entity = entities[m_id]
147
            else:
148
                entity, created = Entity.objects.get_or_create(entityid=m_id)
149
150
            entityid = entity.entityid
151
            name = entity.name
152
            registration_authority = entity.registration_authority
153
            certstats = entity.certstats
154
            display_protocols = entity.display_protocols
155
 
156
            entity_from_xml = self._metadata.get_entity(m_id, False)
157
            entity.process_metadata(False, entity_from_xml, cached_entity_types)
158
159
            if created or entity.has_changed(entityid, name, registration_authority, certstats, display_protocols):
160
                entities_to_update.append(entity)
161
162
            entities_to_add.append(entity)
163
164
        self._update_entities(entities_to_update, entities_to_add)
165
        return len(entities_to_update) 
166
167
    @staticmethod
168
    def _daterange(start_date, end_date):
169
        for n in range(int ((end_date - start_date).days + 1)):
170
            yield start_date + timedelta(n)
171
172
    def compute_new_stats(self):
173
        if not self._metadata: return ([], [])
174
        entities_from_xml = self._metadata.get_entities()
175
176
        entities = Entity.objects.filter(entityid__in=entities_from_xml)
177
        entities = entities.prefetch_related('types')
178
        Entity_Federations.objects.filter(federation=self)
179
180
        try:
181
            first_date = EntityStat.objects.filter(federation=self).aggregate(Max('time'))['time__max']
182
            if not first_date:
183
                raise Exception('Not able to find statistical data in the DB.')
184
        except Exception:
185
            first_date = datetime(2010, 1, 1)
186
            first_date = pytz.utc.localize(first_date)
187
      
188
        for curtimestamp in self._daterange(first_date, timezone.now()):
189
            computed = {}
190
            not_computed = []
191
            entity_stats = []
192
            for feature in stats['features'].keys():
193
                fun = getattr(self, 'get_%s' % feature, None)
194
    
195
                if callable(fun):
196
                    stat = EntityStat()
197
                    stat.feature = feature
198
                    stat.time = curtimestamp
199
                    stat.federation = self
200
                    stat.value = fun(entities, stats['features'][feature], curtimestamp)
201
                    entity_stats.append(stat)
202
                    computed[feature] = stat.value
203
                else:
204
                    not_computed.append(feature)
205
206
            from_time = datetime.combine(curtimestamp, time.min) 
207
            if timezone.is_naive(from_time):
208
                from_time = pytz.utc.localize(from_time)
209
            to_time = datetime.combine(curtimestamp, time.max)
210
            if timezone.is_naive(to_time):
211
                to_time = pytz.utc.localize(to_time)
212
213
            EntityStat.objects.filter(federation=self, time__gte=from_time, time__lte=to_time).delete()
214
            EntityStat.objects.bulk_create(entity_stats)
215
216
        return (computed, not_computed)
217
218
    def process_metadata_entities(self, request=None, federation_slug=None):
219
        if not self._metadata: return
220
        entities_from_xml = self._metadata.get_entities()
221
        removed = self._remove_deleted_entities(entities_from_xml)
222
223
        entities = {}
224
        db_entities = Entity.objects.filter(entityid__in=entities_from_xml)
225
        db_entities = db_entities.prefetch_related('types', 'entity_categories')
226
227
        for entity in db_entities.all():
228
            entities[entity.entityid] = entity
229
230
        if request and federation_slug:
231
            request.session['%s_num_entities' % federation_slug] = len(entities_from_xml)
232
            request.session['%s_cur_entities' % federation_slug] = 0
233
            request.session['%s_process_done' % federation_slug] = False
234
            request.session.save()
235
236
        updated = self._add_new_entities(entities, entities_from_xml, request, federation_slug)
237
238
        if request and federation_slug:
239
            request.session['%s_process_done' % federation_slug] = True
240
            request.session.save()
241
242
        return removed, updated
243
244
    def get_absolute_url(self):
245
        return reverse('federation_view', args=[self.slug])
246
247
    @classmethod
248
    def get_sp(cls, entities, xml_name, ref_date=None):
249
        if ref_date and ref_date < pytz.utc.localize(datetime.now() - timedelta(days = 1)):
250
            selected = entities.filter(types__xmlname=xml_name, entity_federations__registration_instant__lt = ref_date)
251
        else:
252
            selected = entities.filter(types__xmlname=xml_name)
253
        return len(selected)
254
255
    @classmethod
256
    def get_idp(cls, entities, xml_name, ref_date=None):
257
        if ref_date and ref_date < pytz.utc.localize(datetime.now() - timedelta(days = 1)):
258
            selected = entities.filter(types__xmlname=xml_name, entity_federations__registration_instant__lt = ref_date)
259
        else:
260
            selected = entities.filter(types__xmlname=xml_name)
261
        return len(selected)
262
263
    @classmethod
264
    def get_aa(cls, entities, xml_name, ref_date=None):
265
        if ref_date and ref_date < pytz.utc.localize(datetime.now() - timedelta(days = 1)):
266
            selected = entities.filter(types__xmlname=xml_name, entity_federations__registration_instant__lt = ref_date)
267
        else:
268
            selected = entities.filter(types__xmlname=xml_name)
269
        return len(selected)
270
271
    def get_sp_saml1(self, entities, xml_name, ref_date = None):
272
        return self.get_stat_protocol(entities, xml_name, 'SPSSODescriptor', ref_date)
273
274
    def get_sp_saml2(self, entities, xml_name, ref_date = None):
275
        return self.get_stat_protocol(entities, xml_name, 'SPSSODescriptor', ref_date)
276
277
    def get_sp_shib1(self, entities, xml_name, ref_date = None):
278
        return self.get_stat_protocol(entities, xml_name, 'SPSSODescriptor', ref_date)
279
280
    def get_idp_saml1(self, entities, xml_name, ref_date = None):
281
        return self.get_stat_protocol(entities, xml_name, 'IDPSSODescriptor', ref_date)
282
283
    def get_idp_saml2(self, entities, xml_name, ref_date = None):
284
        return self.get_stat_protocol(entities, xml_name, 'IDPSSODescriptor', ref_date)
285
286
    def get_idp_shib1(self, entities, xml_name, ref_date = None):
287
        return self.get_stat_protocol(entities, xml_name, 'IDPSSODescriptor', ref_date)
288
289
    def get_stat_protocol(self, entities, xml_name, service_type, ref_date):
290
        if ref_date and ref_date < pytz.utc.localize(datetime.now() - timedelta(days = 1)):
291
            selected = entities.filter(types__xmlname=service_type, _display_protocols__contains=xml_name, entity_federations__registration_instant__lt = ref_date)
292
        else:
293
            selected = entities.filter(types__xmlname=service_type, _display_protocols__contains=xml_name)
294
        return len(selected)
295
296
    def can_edit(self, user, delete):
297
        if user.is_superuser:
298
            return True
299
300
        permission = 'delete_federation' if delete else 'change_federation'
301
        if user.has_perm('metadataparser.%s' % permission) and user in self.editor_users.all():
302
            return True
303
        return False
304
305
@receiver(pre_save, sender=Federation, dispatch_uid='federation_pre_save')
306
def federation_pre_save(sender, instance, **kwargs):
307
    # Skip pre_save if only file name is saved
308
    if kwargs.has_key('update_fields') and kwargs['update_fields'] == set(['file']):
309
        return
310
311
    #slug = slugify(unicode(instance.name))[:200]
312
    #if instance.file_url and instance.file_url != '':
313
    #    try:
314
    #        instance.fetch_metadata_file(slug)
315
    #    except Exception, e:
316
    #        pass
317
318
    if instance.name:
319
        instance.slug = slugify(unicode(instance))[:200]
320
321
322
@receiver(pre_save, sender=Entity, dispatch_uid='entity_pre_save')
323
def entity_pre_save(sender, instance, **kwargs):
324
    #if refetch and instance.file_url:
325
    #    slug = slugify(unicode(instance.name))[:200]
326
    #    instance.fetch_metadata_file(slug)
327
    #    instance.process_metadata()
328
    pass
329