Completed
Push — master ( 8d75c3...1ca7d8 )
by Andrea
02:35 queued 01:18
created

stats_chart()   C

Complexity

Conditions 7

Size

Total Lines 32

Duplication

Lines 0
Ratio 0 %

Importance

Changes 2
Bugs 0 Features 0
Metric Value
c 2
b 0
f 0
dl 0
loc 32
rs 5.5
cc 7
1
#################################################################
2
# MET v2 Metadate Explorer Tool
3
#
4
# This Software is Open Source. See License: https://github.com/TERENA/met/blob/master/LICENSE.md
5
# Copyright (c) 2012, TERENA All rights reserved.
6
#
7
# This Software is based on MET v1 developed for TERENA by Yaco Sistemas, http://www.yaco.es/
8
# MET v2 was developed for TERENA by Tamim Ziai, DAASI International GmbH, http://www.daasi.de
9
# Current version of MET has been revised for performance improvements by Andrea Biancini,
10
# Consortium GARR, http://www.garr.it
11
#########################################################################################
12
13
import re, time
14
import pytz
15
import simplejson as json
16
from urllib import unquote
17
from datetime import datetime
18
from dateutil import tz
19
20
from django.conf import settings
21
from django.db.models import Count
22
from django.contrib import messages
23
from django.contrib.auth import logout
24
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
25
from django.core.urlresolvers import reverse
26
from django.http import HttpResponse, HttpResponseRedirect, HttpResponseBadRequest
27
from django.shortcuts import render_to_response, get_object_or_404
28
from django.template import RequestContext
29
from django.utils.translation import ugettext_lazy as _
30
from django.utils import timezone
31
32
from chartit import DataPool, Chart
33
34
from met.metadataparser.decorators import user_can_edit
35
from met.metadataparser.models import Federation, Entity, EntityStat, TOP_LENGTH, FEDERATION_TYPES
36
from met.metadataparser.forms import (FederationForm, EntityForm, EntityCommentForm,
37
                                      EntityProposalForm, ServiceSearchForm, ChartForm)
38
39
from met.metadataparser.summary_export import export_summary
40
from met.metadataparser.query_export import export_query_set
41
from met.metadataparser.entity_export import export_entity
42
from met.metadataparser.xmlparser import DESCRIPTOR_TYPES
43
from met.metadataparser.utils import send_mail
44
45
if settings.PROFILE:
46
    from silk.profiling.profiler import silk_profile as profile
47
else:
48
    from met.metadataparser.templatetags.decorators import noop_decorator as profile
49
50
RESCUE_SLASH = re.compile(r"^(http(?:|s):/)([^/])")
51
52
def increment_current_toplength(request):
53
    current_top_length = request.session.get('currentTopLength', TOP_LENGTH)
54
    current_top_length += TOP_LENGTH
55
56
    if current_top_length > Entity.objects.all().count():
57
        current_top_length -= TOP_LENGTH
58
59
    request.session['currentTopLength'] = current_top_length
60
    return HttpResponseRedirect(reverse('index'))
61
    
62
def decrement_current_toplength(request):
63
    current_top_length = request.session.get('currentTopLength', TOP_LENGTH)
64
    current_top_length -= TOP_LENGTH
65
66
    if current_top_length <= 0:
67
        current_top_length = TOP_LENGTH
68
69
    request.session['currentTopLength'] = current_top_length
70
    return HttpResponseRedirect(reverse('index'))
71
72
def _index_export(export, export_format, objects):
73
    counters = (
74
                ('all', {}),
75
                ('IDPSSO', {'types__xmlname': 'IDPSSODescriptor'}),
76
                ('SPSSO', {'types__xmlname': 'SPSSODescriptor'}),
77
               )
78
79
    if not export in ['interfederations', 'federations', 'most_federated_entities']:
80
        return HttpResponseBadRequest('Not valid export query')
81
82
    if export == 'most_federated_entities':
83
        return export_query_set(export_format, objects['most_federated_entities'],
84
                                'most_federated_entities', ('entityid', 'types', 'name', 'federations'))
85
    else:
86
        return export_summary(export_format, objects[export],
87
                              'entity_set', '%s_summary' % export,
88
                              counters)
89
90
@profile(name='Index page')
91
def index(request):
92
    ff = Federation.objects.all()
93
    federations = []
94
    interfederations = []
95
    for f in ff:
96
        if f.is_interfederation:
97
            interfederations.append(f)
98
        else:
99
            federations.append(f)
100
101
    cc = Entity.objects.values('federations__id', 'types__xmlname').annotate(Count('federations__id'), Count('types__xmlname'))
102
    counts = {}
103
    for curtype in DESCRIPTOR_TYPES:
104
        counts[curtype] = []
105
        for c in cc:
106
            if c['types__xmlname'] == curtype:
107
                counts[curtype].append(c)
108
    counts['All'] = Entity.objects.values('federations__id').annotate(Count('federations__id'))
109
110
    totals = Entity.objects.values('types__xmlname').annotate(Count('types__xmlname'))
111
112
    # Entities with count how many federations belongs to, and sorted by most first
113
    current_top_length = request.session.get('currentTopLength', TOP_LENGTH)
114
    most_federated_entities = Entity.get_most_federated_entities(maxlength=current_top_length, cache_expire=24*60*60)
115
116
    params = {
117
       'settings': settings,
118
       'interfederations': interfederations,
119
       'federations': federations,
120
       'entity_types': DESCRIPTOR_TYPES,
121
       'federation_path': request.path,
122
       'counts': counts,
123
       'totals': totals,
124
       'most_federated_entities': most_federated_entities,
125
    }
126
127
    export = request.GET.get('export', None)
128
    export_format = request.GET.get('format', None)
129
    if export and export_format:
130
        return _index_export(export, export_format, params)
131
    
132
    return render_to_response('metadataparser/index.html', params, context_instance=RequestContext(request))
133
134
def _paginate_fed(ob_entities, page):
135
    paginator = Paginator(ob_entities, 20)
136
137
    try:
138
        ob_entities = paginator.page(page)
139
    except PageNotAnInteger:
140
        ob_entities = paginator.page(1)
141
    except EmptyPage:
142
        ob_entities = paginator.page(paginator.num_pages)
143
144
    adjacent_pages = 5
145
    page_range = [n for n in \
146
                  range(ob_entities.number - adjacent_pages, ob_entities.number + adjacent_pages + 1) \
147
                  if n > 0 and n <= ob_entities.paginator.num_pages]
148
149
    return {
150
        'page_range': page_range,
151
        'cur_page_number': ob_entities.number,
152
        'num_pages': ob_entities.paginator.num_pages,
153
    }
154
155
@profile(name='Federation view')
156
def federation_view(request, federation_slug=None):
157
    if federation_slug:
158
        request.session['%s_process_done' % federation_slug] = False
159
        request.session['%s_num_entities' % federation_slug] = 0
160
        request.session['%s_cur_entities' % federation_slug] = 0
161
        request.session.save()
162
163
    federation = get_object_or_404(Federation, slug=federation_slug)
164
165
    entity_type = None
166
    if request.GET and 'entity_type' in request.GET:
167
        entity_type = request.GET['entity_type']
168
        ob_entities = Entity.objects.filter(federations__id=federation.id).filter(types__xmlname=entity_type)
169
    else:
170
        ob_entities = Entity.objects.filter(federations__id=federation.id)
171
    
172
    ob_entities = ob_entities.prefetch_related('types', 'federations')
173
    pagination = _paginate_fed(ob_entities, request.GET.get('page'))
174
175
    entities = []
176
    for entity in ob_entities:
177
        entities.append({
178
            'entityid': entity.entityid,
179
            'name': entity.name,
180
            'absolute_url': entity.get_absolute_url(),
181
            'types': [unicode(item) for item in entity.types.all()],
182
            'federations': [(unicode(item.name), item.get_absolute_url()) for item in entity.federations.all()],
183
        })
184
185
    if 'format' in request.GET:
186
        return export_query_set(request.GET.get('format'), entities,
187
                                'entities_search_result', ('entityid', 'types', 'federations'))
188
189
    context = RequestContext(request)
190
    user = context.get('user', None)
191
    add_entity = user and user.has_perm('metadataparser.add_federation')
192
    pie_chart = fed_pie_chart(request, federation.id)
193
194
    return render_to_response('metadataparser/federation_view.html',
195
            {'settings': settings,
196
             'federation': federation,
197
             'entity_types': DESCRIPTOR_TYPES,
198
             'entity_type': entity_type or 'All',
199
             'fed_types': dict(FEDERATION_TYPES),
200
             'entities': entities,
201
             'show_filters': True,
202
             'add_entity': add_entity,
203
             'lang': request.GET.get('lang', 'en'),
204
             'update_entities': request.GET.get('update', 'false'),
205
             'statcharts': [pie_chart],
206
             'pagination': pagination,
207
            }, context_instance=context)
208
209
210
@user_can_edit(Federation)
211
def federation_edit_post(request, federation, form):
212
    modify = True if federation else False
213
    form.save()
214
215
    if not modify:
216
        form.instance.editor_users.add(request.user)
217
    if 'file' in form.changed_data or 'file_url' in form.changed_data:
218
        form.instance.process_metadata()
219
        #form.instance.process_metadata_entities(request=request)
220
221
    messages.success(request, _('Federation %s successfully' % 'modified' if modify else 'created'))
222
    return HttpResponseRedirect(form.instance.get_absolute_url() + '?update=true')
223
224
225
@user_can_edit(Federation)
226
def federation_edit(request, federation_slug=None):
227
    federation = get_object_or_404(Federation, slug=federation_slug) if federation_slug else None
228
229
    if request.method == 'POST':
230
        form = FederationForm(request.POST, request.FILES, instance=federation)
231
        if not form.is_valid():
232
            messages.error(request, _('Please correct the errors indicated below'))
233
        else:
234
            return federation_edit_post(request, federation, form)
235
    else:
236
        form = FederationForm(instance=federation)
237
238
    context = RequestContext(request)
239
    user = context.get('user', None)
240
    delete_federation = user and user.has_perm('metadataparser.delete_federation')
241
    return render_to_response('metadataparser/federation_edit.html',
242
                              {'settings': settings, 'form': form, 'delete_federation': delete_federation},
243
                              context_instance=RequestContext(request))
244
245
246
@user_can_edit(Federation)
247
def federation_update_entities(request, federation_slug=None):
248
    federation = get_object_or_404(Federation, slug=federation_slug)
249
    federation.process_metadata_entities(request=request, federation_slug=federation_slug)
250
251
    messages.success(request, _('Federation entities updated succesfully'))
252
    return HttpResponse("Done. All entities updated.", content_type='text/plain')
253
254
255
def entityupdate_progress(request, federation_slug=None):
256
    data = { 'done': False }
257
    if federation_slug:
258
        data = { 'done': request.session.get('%s_process_done' % federation_slug, False),
259
                 'tot': request.session.get('%s_num_entities' % federation_slug, 0),
260
                 'num': request.session.get('%s_cur_entities' % federation_slug, 0) }
261
262
    return HttpResponse(json.dumps(data), content_type='application/javascript')
263
264
265
@user_can_edit(Federation, True)
266
def federation_delete(request, federation_slug):
267
    federation = get_object_or_404(Federation, slug=federation_slug)
268
269
    for entity in federation.entity_set.all():
270
        if len(entity.federations.all()) == 1:
271
            entity.delete()
272
273
    messages.success(request,
274
                     _(u"%(federation)s federation was deleted successfully"
275
                     % {'federation': unicode(federation)}))
276
    federation.delete()
277
    return HttpResponseRedirect(reverse('index'))
278
279
280
@profile(name='Index charts')
281
def federation_charts(request, federation_slug=None):
282
    if federation_slug is None:
283
        federation = None
284
    else:
285
        federation = get_object_or_404(Federation, slug=federation_slug)
286
287
    if request.method == 'POST':
288
        form = ChartForm(request.POST, request.FILES, instance=federation)
289
290
        if form.is_valid():
291
            stats_config_dict = getattr(settings, "STATS")
292
            service_terms = stats_config_dict['statistics']['entity_by_type']['terms']
293
            protocol_terms = stats_config_dict['statistics']['entity_by_protocol']['terms']
294
            
295
            protocols = stats_config_dict['protocols']
296
297
            from_time = datetime.fromordinal(form.cleaned_data['fromDate'].toordinal())
298
            if timezone.is_naive(from_time):
299
                from_time = pytz.utc.localize(from_time)
300
            to_time = datetime.fromordinal(form.cleaned_data['toDate'].toordinal() + 1)
301
            if timezone.is_naive(to_time):
302
                to_time = pytz.utc.localize(to_time)
303
304
            service_stats = EntityStat.objects.filter(  federation=federation \
305
                                              , feature__in = service_terms \
306
                                              , time__gte = from_time \
307
                                              , time__lte = to_time).order_by("time")
308
309
            protocol_stats = EntityStat.objects.filter(  federation=federation \
310
                                              , feature__in = protocol_terms \
311
                                              , time__gte = from_time \
312
                                              , time__lte = to_time).order_by("time")
313
314
            s_chart = stats_chart(stats_config_dict, request, service_stats, 'entity_by_type')
315
316
            p_chart = stats_chart(stats_config_dict, request, protocol_stats, 'entity_by_protocol', protocols)
317
318
            return render_to_response('metadataparser/federation_chart.html',
319
                                      {'form': form,
320
                                       'statcharts': [s_chart, p_chart],
321
                                      },
322
                                      context_instance=RequestContext(request))
323
324
        else:
325
            messages.error(request, _('Please correct the errors indicated'
326
                                      ' below'))
327
    else:
328
        form = ChartForm(instance=federation)
329
330
    return render_to_response('metadataparser/federation_chart.html',
331
                              {'settings': settings, 'form': form},
332
                              context_instance=RequestContext(request))
333
334
335
def stats_chart(stats_config_dict, request, stats, entity, protocols=None):
336
    terms = stats_config_dict['statistics'][entity]['terms']
337
    title = stats_config_dict['statistics'][entity]['title']
338
    x_title = stats_config_dict['statistics'][entity]['x_title']
339
    y_title = stats_config_dict['statistics'][entity]['y_title']
340
    chart_type = 'column'
341
    stacking = True
342
    term_names = stats_config_dict['feature_names']
343
    time_format = stats_config_dict['time_format']
344
    statdata = _create_statdata('bar', stats, terms, term_names)
345
346
    series_options = []
347
    for stack in range(len(protocols) if protocols else 1):
348
        for term in terms:
349
            if not protocols or term.endswith(protocols[stack]):
350
                series_options += \
351
                  [{'options':{
352
                      'type': chart_type,
353
                      'stacking': stacking,
354
                      'stack': stack,
355
                    },
356
                    'terms':{
357
                      'time_%s' %term: [{term_names[term]: {'stack': stack, }}],
358
                    }}]
359
360
    chart_options = _get_chart_options('bar', title, x_title, y_title)
361
362
    return Chart(
363
        datasource = statdata,
364
        series_options = series_options,
365
        chart_options = chart_options, 
366
        x_sortf_mapf_mts=(None, lambda i: datetime.fromtimestamp(time.mktime(i.replace(tzinfo=tz.gettz('UTC')).astimezone(tz.tzlocal()).timetuple())).strftime(time_format), False)
367
    )
368
369
370
371
def _create_statdata(chart_type, stats, terms=None, term_names=None):
372
    if chart_type == 'bar':
373
        statdata = DataPool(
374
           series=[{'options': {
375
                       'source': stats.filter(feature=term)},
376
                       'legend_by': 'feature',
377
                       'terms': [{'time_%s' %term :'time'}, 
378
                                 {term_names[term] : 'value', 'name': 'feature'}]
379
                    } for term in terms]
380
        )
381
    elif chart_type == 'pie':
382
        statdata = DataPool(
383
           series=[{'options': { 'source': stats },
384
                    'legend_by': 'feature',
385
                    'terms': ['feature', 'value'],
386
                  }]
387
        )
388
    else:
389
        statdata = None
390
391
    return statdata
392
393
def _get_chart_options(chart_type, title=None, x_title=None, y_title=None):
394
    if chart_type == 'bar':
395
        chart_options = {'title': { 'text': title },
396
           'xAxis': {
397
                'title': { 'text': x_title },
398
                'labels': {
399
                   'rotation': -45,
400
                   'align': 'right'
401
                },
402
                'max': 10,
403
            },
404
           'yAxis': {
405
                'title': { 'text': y_title },
406
                'minorTickInterval': 'auto'
407
            },
408
           'credits': { 'enabled': False },
409
           'scrollbar': { 'enabled': True },
410
           'zoomType': 'xy',
411
        }
412
    elif chart_type == 'pie':
413
        chart_options = {
414
            'title': { 'text': ' ' },
415
            'credits': { 'enabled': False}
416
        }
417
    else:
418
        chart_options = None
419
420
    return chart_options
421
422
def fed_pie_chart(request, federation_id):
423
    stats_config_dict = getattr(settings, "STATS")
424
    terms = stats_config_dict['statistics']['entity_by_type']['terms']
425
    stats = EntityStat.objects.filter(federation = federation_id, \
426
                                      feature__in = terms).order_by('-time')[0:len(terms)]
427
    statdata = _create_statdata('pie', stats)
428
    series_options = \
429
        [{'options': { 'type': 'pie', 'stacking': False, 'size': '70%' },
430
         'terms':{ 'feature': [ 'value' ] }}]
431
    chart_options = _get_chart_options('pie')
432
433
    return Chart(
434
        datasource = statdata,
435
        series_options = series_options,
436
        chart_options = chart_options,
437
    )
438
439
440
441
@profile(name='Entity view')
442
def entity_view(request, entityid):
443
    entityid = unquote(entityid)
444
    entityid = RESCUE_SLASH.sub("\\1/\\2", entityid)
445
446
    entity = get_object_or_404(Entity, entityid=entityid)
447
448
    if 'federation' in request.GET:
449
        federation = get_object_or_404(Federation, slug=request.GET.get('federation'))
450
        entity.curfed = federation
451
452
    if 'format' in request.GET:
453
        return export_entity(request.GET.get('format'), entity)
454
455
    if 'viewxml' in request.GET:
456
        serialized = entity.xml
457
        response = HttpResponse(serialized, content_type='application/xml')
458
        return response
459
460
    return render_to_response('metadataparser/entity_view.html',
461
            {'settings': settings,
462
             'entity': entity,
463
             'lang': request.GET.get('lang', 'en') 
464
            }, context_instance=RequestContext(request))
465
466
467
@user_can_edit(Entity)
468
def entity_edit_post(request, form, federation, entity):
469
    form.save()
470
    if federation and not federation in form.instance.federations.all():
471
        form.instance.federations.add(federation)
472
        form.instance.save()
473
474
    if entity:
475
        messages.success(request, _('Entity modified successfully'))
476
    else:
477
        messages.success(request, _('Entity created successfully'))
478
479
    return HttpResponseRedirect(form.instance.get_absolute_url())
480
481
482
483
@user_can_edit(Entity)
484
def entity_edit(request, federation_slug=None, entity_id=None):
485
    entity = None
486
    federation = None
487
    if federation_slug:
488
        federation = get_object_or_404(Federation, slug=federation_slug)
489
        if entity_id:
490
            entity = get_object_or_404(Entity, id=entity_id,
491
                                       federations__id=federation.id)
492
    if entity_id and not federation_slug:
493
        entity = get_object_or_404(Entity, id=entity_id)
494
495
    if request.method == 'POST':
496
        form = EntityForm(request.POST, request.FILES, instance=entity)
497
        if form.is_valid():
498
            return entity_edit_post(request, form, federation, entity)
499
        else:
500
            messages.error(request, _('Please correct the errors indicated below'))
501
    else:
502
        form = EntityForm(instance=entity)
503
504
    return render_to_response('metadataparser/entity_edit.html',
505
                              {'settings': settings,
506
                               'form': form,
507
                               'federation': federation},
508
                              context_instance=RequestContext(request))
509
510
511
@user_can_edit(Entity, True)
512
def entity_delete(request, entity_id):
513
    entity = get_object_or_404(Entity, id=entity_id)
514
    messages.success(request,
515
                     _(u"%(entity)s entity was deleted successfully"
516
                     % {'entity': unicode(entity)}))
517
    entity.delete()
518
    return HttpResponseRedirect(reverse('index'))
519
520
521
def entity_comment(request, federation_slug=None, entity_id=None):
522
    entity = None
523
    federation = None
524
    if federation_slug:
525
        federation = get_object_or_404(Federation, slug=federation_slug)
526
        if entity_id:
527
            entity = get_object_or_404(Entity, id=entity_id,
528
                                       federations__id=federation.id)
529
    if entity_id and not federation_slug:
530
        entity = get_object_or_404(Entity, id=entity_id)
531
532
    if request.method == 'POST':
533
        form = EntityCommentForm(request.POST, request.FILES, instance=entity)
534
        if form.is_valid():
535
            mail_config = getattr(settings, "MAIL_CONFIG")
536
            try:
537
                subject = mail_config['comment_subject'] %entity
538
                send_mail(form.data['email'], subject, form.data['comment'])
539
                messages.success(request, _('Comment posted successfully'))
540
            except Exception, errorMessage:
541
                messages.error(request, _('Comment could not be posted successfully: %s' %errorMessage))
542
543
            return HttpResponseRedirect(form.instance.get_absolute_url())
544
545
        else:
546
            messages.error(request, _('Please correct the errors indicated'
547
                                      ' below'))
548
    else:
549
        form = EntityCommentForm(instance=entity)
550
551
    return render_to_response('metadataparser/entity_comment.html',
552
                              {'settings': settings,
553
                               'form': form,
554
                              },
555
                              context_instance=RequestContext(request))
556
557
558
def entity_proposal(request, federation_slug=None, entity_id=None):
559
    entity = None
560
    federation = None
561
    if federation_slug:
562
        federation = get_object_or_404(Federation, slug=federation_slug)
563
        if entity_id:
564
            entity = get_object_or_404(Entity, id=entity_id,
565
                                       federations__id=federation.id)
566
    if entity_id and not federation_slug:
567
        entity = get_object_or_404(Entity, id=entity_id)
568
569
    if request.method == 'POST':
570
        form = EntityProposalForm(request.POST, request.FILES, instance=entity)
571
     
572
        if form.is_valid():
573
            mail_config = getattr(settings, "MAIL_CONFIG")
574
            try:
575
                subject = mail_config['proposal_subject'] %entity
576
                my_dict = dict(form.data.iterlists())
577
                body = mail_config['proposal_body'] % (entity, ', '.join(my_dict['federations']), form.data['comment'])
578
                send_mail(form.data['email'], subject, body)
579
                messages.success(request, _('Proposal posted successfully'))
580
            except Exception, errorMessage:
581
                messages.error(request, _('Proposal could not be posted successfully: %s' %errorMessage))
582
583
            return HttpResponseRedirect(form.instance.get_absolute_url())
584
585
        else:
586
            messages.error(request, _('Please correct the errors indicated'
587
                                      ' below'))
588
    else:
589
        form = EntityProposalForm(instance=entity)
590
591
    return render_to_response('metadataparser/entity_proposal.html',
592
                              {'settings': settings,
593
                               'form': form,
594
                              },
595
                              context_instance=RequestContext(request))
596
597
def search_service(request):
598
    filters = {}
599
    objects = []
600
601
    if 'entityid' in request.GET:
602
        form = ServiceSearchForm(request.GET)
603
        if form.is_valid():
604
            entityid = form.cleaned_data['entityid']
605
            entityid = entityid.strip()
606
            filters['entityid__icontains'] = entityid
607
    else:
608
        form = ServiceSearchForm()
609
    entity_type = request.GET.get('entity_type', None)
610
611
    if entity_type:
612
        filters['entity_type'] = entity_type
613
614
    if filters:
615
        objects = Entity.objects.filter(**filters)
616
617
    if objects and 'format' in request.GET.keys():
618
        return export_query_set(request.GET.get('format'), objects,
619
                                'entities_search_result', ('entityid', 'types', 'federations'))
620
621
    entities = []
622
    for entity in objects:
623
        entities.append({
624
            'entityid': entity.entityid,
625
            'name': entity.name,
626
            'absolute_url': entity.get_absolute_url(),
627
            'types': [unicode(item) for item in entity.types.all()],
628
            'federations': [(unicode(item.name), item.get_absolute_url()) for item in entity.federations.all()],
629
        })
630
631
    return render_to_response('metadataparser/service_search.html',
632
        {'settings': settings,
633
         'searchform': form,
634
         'object_list': entities,
635
         'show_filters': False,
636
        }, context_instance=RequestContext(request))
637
638
def met_logout(request):
639
    logout(request)
640
    return HttpResponseRedirect(request.GET.get("next", "/"))
641