|
1
|
|
|
################################################################# |
|
2
|
|
|
# MET v2 Metadate Explorer Tool |
|
3
|
|
|
# |
|
4
|
|
|
# This Software is Open Source. See License: https://github.com/TERENA/met/blob/master/LICENSE.md |
|
5
|
|
|
# Copyright (c) 2012, TERENA All rights reserved. |
|
6
|
|
|
# |
|
7
|
|
|
# This Software is based on MET v1 developed for TERENA by Yaco Sistemas, http://www.yaco.es/ |
|
8
|
|
|
# MET v2 was developed for TERENA by Tamim Ziai, DAASI International GmbH, http://www.daasi.de |
|
9
|
|
|
# Current version of MET has been revised for performance improvements by Andrea Biancini, |
|
10
|
|
|
# Consortium GARR, http://www.garr.it |
|
11
|
|
|
######################################################################################### |
|
12
|
|
|
|
|
13
|
|
|
import logging |
|
14
|
|
|
from datetime import date |
|
15
|
|
|
|
|
16
|
|
|
from django.conf import settings |
|
17
|
|
|
from django.db.models import Count |
|
18
|
|
|
|
|
19
|
|
|
from met.metadataparser.utils import send_mail, send_slack |
|
20
|
|
|
from met.metadataparser.models import Federation, Entity |
|
21
|
|
|
|
|
22
|
|
|
if settings.PROFILE: |
|
23
|
|
|
from silk.profiling.profiler import silk_profile as profile |
|
24
|
|
|
else: |
|
25
|
|
|
from met.metadataparser.templatetags.decorators import noop_decorator as profile |
|
26
|
|
|
|
|
27
|
|
|
def _send_message_via_email_and_slack(error_msg, federation, logger=None): |
|
28
|
|
|
mail_config_dict = getattr(settings, "MAIL_CONFIG") |
|
29
|
|
|
try: |
|
30
|
|
|
subject = mail_config_dict['refresh_subject'] % federation |
|
31
|
|
|
from_address = mail_config_dict['from_email_address'] |
|
32
|
|
|
send_mail(from_address, subject, '%s' % error_msg) |
|
33
|
|
|
send_slack('%s' % error_msg) |
|
34
|
|
|
except Exception, errorMessage: |
|
35
|
|
|
log('Message could not be posted successfully: %s' % errorMessage, logger, logging.ERROR) |
|
36
|
|
|
|
|
37
|
|
|
def _fetch_new_metadata_file(federation, logger): |
|
38
|
|
|
try: |
|
39
|
|
|
changed = federation.fetch_metadata_file(federation.slug) |
|
40
|
|
|
return None, changed |
|
41
|
|
|
except Exception, errorMessage: |
|
42
|
|
|
log('%s' % errorMessage, logger, logging.ERROR) |
|
43
|
|
|
return "%s" % errorMessage, False |
|
44
|
|
|
|
|
45
|
|
|
def refresh(fed_name=None, force_refresh=False, logger=None): |
|
46
|
|
|
log('Starting refreshing metadata ...', logger, logging.INFO) |
|
47
|
|
|
|
|
48
|
|
|
log('Deleting all previous entity categories ...', logger, logging.INFO) |
|
49
|
|
|
entities = Entity.objects.all() |
|
50
|
|
|
for entity in entities: |
|
51
|
|
|
entity.entity_categories.all().delete() |
|
52
|
|
|
|
|
53
|
|
|
federations = Federation.objects.all() |
|
54
|
|
|
federations.prefetch_related('etypes', 'federations', 'entity_categories') |
|
55
|
|
|
|
|
56
|
|
|
for federation in federations: |
|
57
|
|
|
if fed_name and federation.slug != fed_name: |
|
58
|
|
|
continue |
|
59
|
|
|
|
|
60
|
|
|
error_msg = None |
|
61
|
|
|
try: |
|
62
|
|
|
log('[%s] Refreshing metadata ...' % federation, logger, logging.INFO) |
|
63
|
|
|
error_msg, data_changed = _fetch_new_metadata_file(federation, logger) |
|
64
|
|
|
|
|
65
|
|
|
if not error_msg and (force_refresh or data_changed): |
|
66
|
|
|
log('[%s] Updating database ...' % federation, logger, logging.INFO) |
|
67
|
|
|
|
|
68
|
|
|
log('[%s] Updating federation ...' % federation, logger, logging.DEBUG) |
|
69
|
|
|
federation.process_metadata() |
|
70
|
|
|
federation.save() |
|
71
|
|
|
|
|
72
|
|
|
log('[%s] Updating federation entities ...' % federation, logger, logging.DEBUG) |
|
73
|
|
|
removed, updated = federation.process_metadata_entities() |
|
74
|
|
|
log('[%s] Removed %s old entities and updated %s entities.' % (federation, removed, updated), logger, logging.INFO) |
|
75
|
|
|
|
|
76
|
|
|
log('[%s] Updating federation file and metadata_data...' % federation, logger, logging.DEBUG) |
|
77
|
|
|
federation.metadata_update = date.today() |
|
78
|
|
|
federation.save(update_fields=['file', 'metadata_update']) |
|
79
|
|
|
log('[%s] Federation update time modified with %s' % (federation, federation.metadata_update), logger, logging.INFO) |
|
80
|
|
|
|
|
81
|
|
|
log('[%s] Updating federation statistics ...' % federation, logger, logging.DEBUG) |
|
82
|
|
|
(computed, not_computed) = federation.compute_new_stats() |
|
83
|
|
|
log('[%s] Computed statistics: %s' % (federation, computed), logger, logging.DEBUG) |
|
84
|
|
|
log('[%s] NOT Computed statistics: %s' % (federation, not_computed), logger, logging.DEBUG) |
|
85
|
|
|
|
|
86
|
|
|
except Exception, e: |
|
87
|
|
|
error_msg = '%s %s' % (error_msg, e) |
|
88
|
|
|
|
|
89
|
|
|
finally: |
|
90
|
|
|
if error_msg: |
|
91
|
|
|
log('Sending following error via email: %s' % error_msg, logger, logging.INFO) |
|
92
|
|
|
_send_message_via_email_and_slack(error_msg, federation, logger) |
|
93
|
|
|
|
|
94
|
|
|
log('Removing entities with no federation associated...', logger, logging.INFO) |
|
95
|
|
|
Entity.objects.all().annotate(federationslength=Count("federations")).filter(federationslength__lte=0).delete() |
|
96
|
|
|
|
|
97
|
|
|
log('Refreshing metadata terminated.', logger, logging.INFO) |
|
98
|
|
|
|
|
99
|
|
|
def log(message, logger=None, severity=logging.INFO): |
|
100
|
|
|
if logger: |
|
101
|
|
|
logger.log(severity, message) |
|
102
|
|
|
else: |
|
103
|
|
|
print(message) |
|
104
|
|
|
|
|
105
|
|
|
|
GEANT /
met
