Completed
Push — master ( d1e883...1f56b4 )
by
unknown
01:28
created

get_daily_data_by_platform()   B

Complexity

Conditions 6

Size

Total Lines 10

Duplication

Lines 10
Ratio 100 %

Importance

Changes 0
Metric Value
cc 6
c 0
b 0
f 0
dl 10
loc 10
rs 8
1
# coding: utf8
2
3
"""
4
This software is licensed under the Apache 2 license, quoted below.
5
6
Copyright 2014 Crystalnix Limited
7
8
Licensed under the Apache License, Version 2.0 (the "License"); you may not
9
use this file except in compliance with the License. You may obtain a copy of
10
the License at
11
12
    http://www.apache.org/licenses/LICENSE-2.0
13
14
Unless required by applicable law or agreed to in writing, software
15
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
17
License for the specific language governing permissions and limitations under
18
the License.
19
"""
20
21
from builtins import range
22
23
from functools import partial
24
from datetime import datetime, timedelta
25
26
from django.conf import settings
27
from django.db import transaction
28
from django.utils import timezone
29
from bitmapist import setup_redis, mark_event, unmark_event, WeekEvents, MonthEvents, DayEvents, HourEvents
30
import pytz
31
32
from omaha.utils import get_id, is_new_install, valuedispatch, redis
33
from omaha import parser
34
from omaha.models import ACTIVE_USERS_DICT_CHOICES, Request, AppRequest, Os, Hw, Event, Version, Channel
35
from sparkle.models import SparkleVersion
36
37
__all__ = ['userid_counting', 'is_user_active']
38
39
host, port, db = settings.CACHES['statistics']['LOCATION'].split(':')
40
setup_redis('default', host, port, db=db)
41
42
43
def userid_counting(userid, apps_list, platform, now=None):
44
    id = get_id(userid)
45
    mark_event('request', id, now=now)
46
    list(map(partial(add_app_statistics, id, platform, now=now), apps_list or []))
47
48
49
def add_app_statistics(userid, platform, app, now=None):
50
    mark = partial(mark_event, now=now)
51
    if not now:
52
        now = timezone.now()
53
    appid = app.get('appid')
54
    version = app.get('version')
55
    channel = parser.get_channel(app)
56
    events = app.findall('event')
57
    nextversion = app.get('nextversion')
58
59
    err_events = filter(lambda x: x.get('eventresult') not in ['1', '2', '3'], events)
60
    if err_events:
61
        return
62
63
    install_event = filter(lambda x: x.get('eventtype') == '2', events)
64
    if is_new_install(appid, userid):
65
        if install_event:
66
            mark('new_install:%s' % appid, userid)
67
            mark('new_install:{}:{}'.format(appid, platform), userid)
68
            redis.setbit("known_users:%s" % appid, userid, 1)
69
            mark('request:{}:{}'.format(appid, nextversion), userid, track_hourly=True)
70
            mark('request:{}:{}:{}'.format(appid, platform, nextversion), userid, track_hourly=True)
71
            mark('request:{}:{}'.format(appid, channel), userid)
72
            return
73
74
    elif userid not in MonthEvents('new_install:{}:{}'.format(appid, platform), year=now.year, month=now.month):
75
        mark('request:%s' % appid, userid)
76
        mark('request:{}:{}'.format(appid, platform), userid)
77
        if nextversion:
78
            mark('request:{}:{}'.format(appid, nextversion), userid, track_hourly=True)
79
            mark('request:{}:{}:{}'.format(appid, platform, nextversion), userid, track_hourly=True)
80
81
    uninstall_event = filter(lambda x: x.get('eventtype') == '4', events)
82
    if uninstall_event:
83
        mark('uninstall:%s' % appid, userid)
84
        mark('uninstall:{}:{}'.format(appid, platform), userid)
85
    update_event = filter(lambda x: x.get('eventtype') == '3', events)
86
    if update_event:
87
        unmark_event('request:{}:{}'.format(appid, version), userid, track_hourly=True)
88
        unmark_event('request:{}:{}:{}'.format(appid, platform, version), userid, track_hourly=True)
89
        mark('request:{}:{}'.format(appid, nextversion), userid, track_hourly=True)
90
        mark('request:{}:{}:{}'.format(appid, platform, nextversion), userid, track_hourly=True)
91
    else:
92
        mark('request:{}:{}'.format(appid, version), userid, track_hourly=True)
93
        mark('request:{}:{}:{}'.format(appid, platform, version), userid, track_hourly=True)
94
    mark('request:{}:{}'.format(appid, channel), userid)
95
96
97
def get_users_statistics_months(app_id, platform=None, year=None, start=1, end=12):
98
    now = timezone.now()
99
    if not year:
100
        year = now.year
101
102
    if platform:
103
        install_event_name = 'new_install:{}:{}'.format(app_id, platform)
104
        update_event_name = 'request:{}:{}'.format(app_id, platform)
105
        uninstall_event_name = 'uninstall:{}:{}'.format(app_id, platform)
106
    else:
107
        install_event_name = 'new_install:%s' % app_id
108
        update_event_name = 'request:%s' % app_id
109
        uninstall_event_name = 'uninstall:%s' % app_id
110
111
    installs_by_month = []
112
    updates_by_month = []
113
    uninstalls_by_month = []
114
    for m in range(start, end + 1):
115
        installs_by_month.append(MonthEvents(install_event_name, year, m))
116
        updates_by_month.append(MonthEvents(update_event_name, year, m))
117
        uninstalls_by_month.append(MonthEvents(uninstall_event_name, year, m))
118
    installs_data = [(datetime(year, start + i, 1).strftime("%Y-%m"), len(e)) for i, e in enumerate(installs_by_month)]
119
    updates_data = [(datetime(year, start + i, 1).strftime("%Y-%m"), len(e)) for i, e in enumerate(updates_by_month)]
120
    res = dict(new=installs_data, updates=updates_data)
121
    if platform != 'mac':
122
        uninstalls_data = [(datetime(year, start + i, 1).strftime("%Y-%m"), len(e)) for i, e in enumerate(uninstalls_by_month)]
123
        res.update(dict(uninstalls=uninstalls_data))
124
    return res
125
126
127
def get_users_statistics_weeks(app_id=None):
128
    now = timezone.now()
129
    event_name = 'request:%s' % app_id if app_id else 'request'
130
    year = now.year
131
    current_week = now.isocalendar()[1]
132
    previous_week = (now - timedelta(weeks=1)).isocalendar()[1]
133
    yesterday = now - timedelta(days=1)
134
    data = [
135
        ('Previous week', len(WeekEvents(event_name, year, previous_week))),
136
        ('Current week', len(WeekEvents(event_name, year, current_week))),
137
        ('Yesterday', len(DayEvents(event_name, year, yesterday.month, yesterday.day))),
138
        ('Today', len(DayEvents(event_name, year, now.month, now.day))),
139
    ]
140
    return data
141
142
143
def get_channel_statistics(app_id, date=None):
144
    if not date:
145
        date = timezone.now()
146
147
    event_name = 'request:{}:{}'
148
    channels = [c.name for c in Channel.objects.all()]
149
    data = [(channel, len(MonthEvents(event_name.format(app_id, channel), date.year, date.month))) for channel in channels]
150
    data = filter(lambda x: x[1], data)
151
    return data
152
153
154
def get_users_versions_by_platform(app_id, platform, date):
155
    if platform == 'win':
156
        versions = [str(v.version) for v in Version.objects.filter_by_enabled(app__id=app_id)]
157
    else:
158
        versions = [str(v.short_version) for v in SparkleVersion.objects.filter_by_enabled(app__id=app_id)]
159
    event_name = 'request:{}:{}:{}'
160
    data = [(v, len(MonthEvents(event_name.format(app_id, platform, v), date.year, date.month))) for v in versions]
161
    return data
162
163
164
def get_users_versions(app_id, date=None):
165
    if not date:
166
        date = timezone.now()
167
168
    win_data = get_users_versions_by_platform(app_id, 'win', date)
169
    win_data = filter(lambda x: x[1], win_data)
170
171
    mac_data = get_users_versions_by_platform(app_id, 'mac', date)
172
    mac_data = filter(lambda x: x[1], mac_data)
173
174
    data = dict(win=dict(win_data), mac=dict(mac_data))
175
176
    return data
177
178
179
180 View Code Duplication
def get_hourly_data_by_platform(app_id, end, n_hours, versions, platform, tz='UTC'):
0 ignored issues
show
Duplication introduced by
This code seems to be duplicated in your project.
Loading history...
181
    tzinfo = pytz.timezone(tz)
182
    start = end - timezone.timedelta(hours=n_hours)
183
    event_name = "request:{}:{}:{}"
184
185
    hours = [datetime(start.year, start.month, start.day, start.hour, tzinfo=pytz.UTC)
186
             + timezone.timedelta(hours=x) for x in range(1, n_hours + 1)]
187
188
    data = [(v, [[hour.astimezone(tzinfo).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
189
                  len(HourEvents.from_date(event_name.format(app_id, platform, v), hour))]
190
                 for hour in hours])
191
            for v in versions]
192
    data = filter(lambda version_data: sum([data[1] for data in version_data[1]]), data)
193
    return dict(data)
194
195
196 View Code Duplication
def get_daily_data_by_platform(app_id, end, n_days, versions, platform):
0 ignored issues
show
Duplication introduced by
This code seems to be duplicated in your project.
Loading history...
197
    start = end - timezone.timedelta(days=n_days)
198
    event_name = "request:{}:{}:{}"
199
200
    days = [start + timezone.timedelta(days=x) for x in range(0, n_days+1)]
201
    data = [(v, [[day.strftime("%Y-%m-%dT00:%M:%S.%fZ"), len(DayEvents.from_date(event_name.format(app_id, platform, v), day))]
202
                 for day in days])
203
            for v in versions]
204
    data = filter(lambda version_data: sum([data[1] for data in version_data[1]]), data)
205
    return dict(data)
206
207
208
def get_users_live_versions(app_id, start, end, tz='UTC'):
209
    win_versions = [str(v.version) for v in Version.objects.filter_by_enabled(app__id=app_id)]
210
    mac_versions = [str(v.short_version) for v in SparkleVersion.objects.filter_by_enabled(app__id=app_id)]
211
212
    if start < timezone.now() - timedelta(days=7):
213
        n_days = (end-start).days
214
        win_data = get_daily_data_by_platform(app_id, end, n_days, win_versions, 'win')
215
        mac_data = get_daily_data_by_platform(app_id, end, n_days, mac_versions, 'mac')
216
    else:
217
        tmp_hours = divmod((end - start).total_seconds(), 60*60)
218
        n_hours = tmp_hours[0]+1
219
        n_hours = int(n_hours)
220
        win_data = get_hourly_data_by_platform(app_id, end, n_hours, win_versions, 'win', tz=tz)
221
        mac_data = get_hourly_data_by_platform(app_id, end, n_hours, mac_versions, 'mac', tz=tz)
222
223
    data = dict(win=win_data, mac=mac_data)
224
225
    return data
226
227
228
@valuedispatch
229
def is_user_active(period, userid):
230
    return False
231
232
233
@is_user_active.register(ACTIVE_USERS_DICT_CHOICES['all'])
234
def _(period, userid):
235
    return True
236
237
238
@is_user_active.register(ACTIVE_USERS_DICT_CHOICES['week'])
239
def _(period, userid):
240
    return get_id(userid) in WeekEvents.from_date('request', timezone.now())
241
242
243
@is_user_active.register(ACTIVE_USERS_DICT_CHOICES['month'])
244
def _(period, userid):
245
    return get_id(userid) in MonthEvents.from_date('request', timezone.now())
246
247
248
def get_kwargs_for_model(cls, obj, exclude=None):
249
    exclude = exclude or []
250
    fields = [(field.name, field.to_python) for field in cls._meta.fields if field.name not in exclude]
251
    kwargs = dict([(i, convert(obj.get(i))) for (i, convert) in fields])
252
    return kwargs
253
254
255
def parse_os(os):
256
    kwargs = get_kwargs_for_model(Os, os, exclude=['id'])
257
    obj, flag = Os.objects.get_or_create(**kwargs)
258
    return obj
259
260
261
def parse_hw(hw):
262
    kwargs = get_kwargs_for_model(Hw, hw, exclude=['id'])
263
    obj, flag = Hw.objects.get_or_create(**kwargs)
264
    return obj
265
266
267
def parse_req(request, ip=None):
268
    kwargs = get_kwargs_for_model(Request, request, exclude=['os', 'hw', 'created', 'id'])
269
    kwargs['ip'] = ip
270
    return Request(**kwargs)
271
272
273
def parse_apps(apps, request):
274
    app_list = []
275
    for app in apps:
276
        events = app.findall('event')
277
278
        if not events:
279
            continue
280
281
        kwargs = get_kwargs_for_model(AppRequest, app, exclude=['request', 'version', 'nextversion', 'id'])
282
        kwargs['version'] = app.get('version') or None
283
        kwargs['nextversion'] = app.get('nextversion') or None
284
        kwargs['tag'] = parser.get_channel(app)
285
        app_req = AppRequest.objects.create(request=request, **kwargs)
286
        event_list = parse_events(events)
287
        app_req.events.add(*event_list)
288
        app_list.append(app_req)
289
    return app_list
290
291
292
def parse_events(events):
293
    res = []
294
    for event in events:
295
        kwargs = get_kwargs_for_model(Event, event)
296
        res.append(Event.objects.create(**kwargs))
297
    return res
298
299
300
@transaction.atomic
301
def collect_statistics(request, ip=None):
302
    userid = request.get('userid')
303
    apps = request.findall('app')
304
305
    if userid:
306
        userid_counting(userid, apps, request.os.get('platform'))
307
308
    if not filter(lambda app: bool(app.findall('event')), apps):
309
        return
310
311
    req = parse_req(request, ip)
312
    req.os = parse_os(request.os)
313
    req.hw = parse_hw(request.hw) if request.get('hw') else None
314
    req.save()
315
316
    parse_apps(apps, req)
317
318