Completed
Push — master ( d57b0b...80936f )
by
unknown
01:57
created

get_users_statistics_months()   D

Complexity

Conditions 8

Size

Total Lines 28

Duplication

Lines 0
Ratio 0 %

Importance

Changes 0
Metric Value
cc 8
c 0
b 0
f 0
dl 0
loc 28
rs 4
1
# coding: utf8
2
3
"""
4
This software is licensed under the Apache 2 license, quoted below.
5
6
Copyright 2014 Crystalnix Limited
7
8
Licensed under the Apache License, Version 2.0 (the "License"); you may not
9
use this file except in compliance with the License. You may obtain a copy of
10
the License at
11
12
    http://www.apache.org/licenses/LICENSE-2.0
13
14
Unless required by applicable law or agreed to in writing, software
15
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
17
License for the specific language governing permissions and limitations under
18
the License.
19
"""
20
21
from builtins import range
22
23
from functools import partial
24
from datetime import datetime, timedelta
25
26
from django.conf import settings
27
from django.db import transaction
28
from django.utils import timezone
29
from bitmapist import setup_redis, mark_event, unmark_event, WeekEvents, MonthEvents, DayEvents, HourEvents
30
import pytz
31
32
from omaha.utils import get_id, is_new_install, valuedispatch, redis
33
from omaha.settings import DEFAULT_CHANNEL
34
from omaha.models import ACTIVE_USERS_DICT_CHOICES, Request, AppRequest, Os, Hw, Event, Version, Channel
35
from sparkle.models import SparkleVersion
36
37
__all__ = ['userid_counting', 'is_user_active']
38
39
host, port, db = settings.CACHES['statistics']['LOCATION'].split(':')
40
setup_redis('default', host, port, db=db)
41
42
43
def userid_counting(userid, apps_list, platform, now=None):
44
    id = get_id(userid)
45
    mark_event('request', id, now=now)
46
    list(map(partial(add_app_statistics, id, platform, now=now), apps_list or []))
47
48
49
def add_app_statistics(userid, platform, app, now=None):
50
    mark = partial(mark_event, now=now)
51
    if not now:
52
        now = timezone.now()
53
    appid = app.get('appid')
54
    version = app.get('version')
55
    channel = app.get('tag') or DEFAULT_CHANNEL
56
    events = app.findall('event')
57
    err_events = filter(lambda x: x.get('eventresult') not in ['1', '2', '3'], events)
58
    if err_events:
59
        return
60
    install_event = filter(lambda x: x.get('eventtype') == '2', events)
61
    if is_new_install(appid, userid):
62
        if install_event and install_event[0].get('eventresult') in ['1', '2', '3']:
63
            mark('new_install:%s' % appid, userid)
64
            mark('new_install:{}:{}'.format(appid, platform), userid)
65
            redis.setbit("known_users:%s" % appid, userid, 1)
66
    elif userid not in MonthEvents('new_install:{}:{}'.format(appid, platform), year=now.year, month=now.month):
67
        mark('request:%s' % appid, userid)
68
        mark('request:{}:{}'.format(appid, platform), userid)
69
70
    uninstall_event = filter(lambda x: x.get('eventtype') == '4', events)
71
    if uninstall_event and uninstall_event[0].get('eventresult') in ['1', '2', '3']:
72
        mark('uninstall:%s' % appid, userid)
73
        mark('uninstall:{}:{}'.format(appid, platform), userid)
74
75
    mark('request:{}:{}'.format(appid, version), userid)
76
    mark('request:{}:{}'.format(appid, channel), userid)
77
    mark('request:{}:{}:{}'.format(appid, platform, version), userid)
78
79
80
def update_live_statistics(userid, apps_list, platform, now=None):
81
    id = get_id(userid)
82
    list(map(partial(add_app_live_statistics, id, platform, now=now), apps_list or []))
83
84
85
def add_app_live_statistics(userid, platform, app, now=None):
86
    mark = partial(mark_event, now=now, track_hourly=True)
87
    unmark = partial(unmark_event, track_hourly=True)
88
    appid = app.get('appid')
89
    version = app.get('version')
90
    events = app.findall('event')
91
    nextversion = app.get('nextversion')
92
93
    install_event = filter(lambda x: x.get('eventtype') == '2', events)
94
    if install_event and install_event[0].get('eventresult') in ['1', '2', '3']:
95
        mark('online:{}:{}'.format(appid, nextversion), userid)
96
        mark('online:{}:{}:{}'.format(appid, platform, nextversion), userid)
97
        return
98
99
    update_event = filter(lambda x: x.get('eventtype') == '3', events)
100
    if update_event and update_event[0].get('eventresult') in ['1', '2', '3']:
101
        unmark('online:{}:{}'.format(appid, version), userid)               # necessary for
102
        unmark('online:{}:{}:{}'.format(appid, platform, version), userid)  # 1 hour interval
103
        mark('online:{}:{}'.format(appid, nextversion), userid)
104
        mark('online:{}:{}:{}'.format(appid, platform, nextversion), userid)
105
        return
106
107
    # updatecheck handling
108
    if version:
109
        mark('online:{}:{}'.format(appid, version), userid)
110
        mark('online:{}:{}:{}'.format(appid, platform, version), userid)
111
112
113
def get_users_statistics_months(app_id, platform=None, year=None, start=1, end=12):
114
    now = timezone.now()
115
    if not year:
116
        year = now.year
117
118
    if platform:
119
        install_event_name = 'new_install:{}:{}'.format(app_id, platform)
120
        update_event_name = 'request:{}:{}'.format(app_id, platform)
121
        uninstall_event_name = 'uninstall:{}:{}'.format(app_id, platform)
122
    else:
123
        install_event_name = 'new_install:%s' % app_id
124
        update_event_name = 'request:%s' % app_id
125
        uninstall_event_name = 'uninstall:%s' % app_id
126
127
    installs_by_month = []
128
    updates_by_month = []
129
    uninstalls_by_month = []
130
    for m in range(start, end + 1):
131
        installs_by_month.append(MonthEvents(install_event_name, year, m))
132
        updates_by_month.append(MonthEvents(update_event_name, year, m))
133
        uninstalls_by_month.append(MonthEvents(uninstall_event_name, year, m))
134
    installs_data = [(datetime(year, start + i, 1).strftime("%Y-%m"), len(e)) for i, e in enumerate(installs_by_month)]
135
    updates_data = [(datetime(year, start + i, 1).strftime("%Y-%m"), len(e)) for i, e in enumerate(updates_by_month)]
136
    res = dict(new=installs_data, updates=updates_data)
137
    if platform != 'mac':
138
        uninstalls_data = [(datetime(year, start + i, 1).strftime("%Y-%m"), len(e)) for i, e in enumerate(uninstalls_by_month)]
139
        res.update(dict(uninstalls=uninstalls_data))
140
    return res
141
142
143
def get_users_statistics_weeks(app_id=None):
144
    now = timezone.now()
145
    event_name = 'request:%s' % app_id if app_id else 'request'
146
    year = now.year
147
    current_week = now.isocalendar()[1]
148
    previous_week = (now - timedelta(weeks=1)).isocalendar()[1]
149
    yesterday = now - timedelta(days=1)
150
    data = [
151
        ('Previous week', len(WeekEvents(event_name, year, previous_week))),
152
        ('Current week', len(WeekEvents(event_name, year, current_week))),
153
        ('Yesterday', len(DayEvents(event_name, year, yesterday.month, yesterday.day))),
154
        ('Today', len(DayEvents(event_name, year, now.month, now.day))),
155
    ]
156
    return data
157
158
159
def get_channel_statistics(app_id, date=None):
160
    if not date:
161
        date = timezone.now()
162
163
    event_name = 'request:{}:{}'
164
    channels = [c.name for c in Channel.objects.all()]
165
    data = [(channel, len(MonthEvents(event_name.format(app_id, channel), date.year, date.month))) for channel in channels]
166
    data = filter(lambda x: x[1], data)
167
    return data
168
169
170
def get_users_versions_by_platform(app_id, platform, date):
171
    if platform == 'win':
172
        versions = [str(v.version) for v in Version.objects.filter_by_enabled(app__id=app_id)]
173
    else:
174
        versions = [str(v.short_version) for v in SparkleVersion.objects.filter_by_enabled(app__id=app_id)]
175
    event_name = 'request:{}:{}:{}'
176
    data = [(v, len(MonthEvents(event_name.format(app_id, platform, v), date.year, date.month))) for v in versions]
177
    return data
178
179
180
def get_users_versions(app_id, date=None):
181
    if not date:
182
        date = timezone.now()
183
184
    win_data = get_users_versions_by_platform(app_id, 'win', date)
185
    win_data = filter(lambda x: x[1], win_data)
186
187
    mac_data = get_users_versions_by_platform(app_id, 'mac', date)
188
    mac_data = filter(lambda x: x[1], mac_data)
189
190
    data = dict(win=dict(win_data), mac=dict(mac_data))
191
192
    return data
193
194
195
196
def get_versions_data_by_platform(app_id, end, n_hours, versions, platform, tz='UTC'):
197
    tzinfo = pytz.timezone(tz)
198
    start = end - timezone.timedelta(hours=n_hours)
199
    event_name = "online:{}:{}:{}"
200
201
    hours = [datetime(start.year, start.month, start.day, start.hour, tzinfo=pytz.UTC)
202
             + timezone.timedelta(hours=x) for x in range(1, n_hours + 1)]
203
204
    data = [(v, [[hour.astimezone(tzinfo).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
205
                  len(HourEvents.from_date(event_name.format(app_id, platform, v), hour))]
206
                 for hour in hours])
207
            for v in versions]
208
    data = filter(lambda version_data: sum([data[1] for data in version_data[1]]), data)
209
    return dict(data)
210
211
212
def get_users_live_versions(app_id, start, end, tz='UTC'):
213
    win_versions = [str(v.version) for v in Version.objects.filter_by_enabled(app__id=app_id)]
214
    mac_versions = [str(v.short_version) for v in SparkleVersion.objects.filter_by_enabled(app__id=app_id)]
215
216
    tmp_hours = divmod((end - start).total_seconds(), 60*60)
217
    n_hours = tmp_hours[0]+1
218
    n_hours = int(n_hours)
219
220
    win_data = get_versions_data_by_platform(app_id, end, n_hours, win_versions, 'win', tz=tz)
221
    mac_data = get_versions_data_by_platform(app_id, end, n_hours, mac_versions, 'mac', tz=tz)
222
223
    data = dict(win=win_data, mac=mac_data)
224
225
    return data
226
227
228
@valuedispatch
229
def is_user_active(period, userid):
230
    return False
231
232
233
@is_user_active.register(ACTIVE_USERS_DICT_CHOICES['all'])
234
def _(period, userid):
235
    return True
236
237
238
@is_user_active.register(ACTIVE_USERS_DICT_CHOICES['week'])
239
def _(period, userid):
240
    return get_id(userid) in WeekEvents.from_date('request', timezone.now())
241
242
243
@is_user_active.register(ACTIVE_USERS_DICT_CHOICES['month'])
244
def _(period, userid):
245
    return get_id(userid) in MonthEvents.from_date('request', timezone.now())
246
247
248
def get_kwargs_for_model(cls, obj, exclude=None):
249
    exclude = exclude or []
250
    fields = [(field.name, field.to_python) for field in cls._meta.fields if field.name not in exclude]
251
    kwargs = dict([(i, convert(obj.get(i))) for (i, convert) in fields])
252
    return kwargs
253
254
255
def parse_os(os):
256
    kwargs = get_kwargs_for_model(Os, os, exclude=['id'])
257
    obj, flag = Os.objects.get_or_create(**kwargs)
258
    return obj
259
260
261
def parse_hw(hw):
262
    kwargs = get_kwargs_for_model(Hw, hw, exclude=['id'])
263
    obj, flag = Hw.objects.get_or_create(**kwargs)
264
    return obj
265
266
267
def parse_req(request, ip=None):
268
    kwargs = get_kwargs_for_model(Request, request, exclude=['os', 'hw', 'created', 'id'])
269
    kwargs['ip'] = ip
270
    return Request(**kwargs)
271
272
273
def parse_apps(apps, request):
274
    app_list = []
275
    for app in apps:
276
        events = app.findall('event')
277
278
        if not events:
279
            continue
280
281
        kwargs = get_kwargs_for_model(AppRequest, app, exclude=['request', 'version', 'nextversion', 'id'])
282
        kwargs['version'] = app.get('version') or None
283
        kwargs['nextversion'] = app.get('nextversion') or None
284
        app_req = AppRequest.objects.create(request=request, **kwargs)
285
        event_list = parse_events(events)
286
        app_req.events.add(*event_list)
287
        app_list.append(app_req)
288
    return app_list
289
290
291
def parse_events(events):
292
    res = []
293
    for event in events:
294
        kwargs = get_kwargs_for_model(Event, event)
295
        res.append(Event.objects.create(**kwargs))
296
    return res
297
298
299
@transaction.atomic
300
def collect_statistics(request, ip=None):
301
    userid = request.get('userid')
302
    apps = request.findall('app')
303
304
    if userid:
305
        userid_counting(userid, apps, request.os.get('platform'))
306
        update_live_statistics(userid, apps, request.os.get('platform'))
307
308
    if not filter(lambda app: bool(app.findall('event')), apps):
309
        return
310
311
    req = parse_req(request, ip)
312
    req.os = parse_os(request.os)
313
    req.hw = parse_hw(request.hw) if request.get('hw') else None
314
    req.save()
315
316
    parse_apps(apps, req)
317
318