add_app_live_statistics() - Code Metrics - Inspection of "Bump version: 0.3.5 → 0.3.6" - Crystalnix/omaha-server - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( 5fa8a3...254aa9 )

by Egor

created 2016-05-11 10:10 UTC

add_app_live_statistics() D

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Metric	Value
cc	8
dl	0
loc	26
rs	4

# coding: utf8

"""
This software is licensed under the Apache 2 license, quoted below.

Copyright 2014 Crystalnix Limited

Licensed under the Apache License, Version 2.0 (the "License"); you may not
use this file except in compliance with the License. You may obtain a copy of
the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations under
the License.
"""

from builtins import range

from functools import partial
from datetime import datetime, timedelta

from django.conf import settings
from django.db import transaction
from django.utils import timezone
from bitmapist import setup_redis, mark_event, unmark_event, WeekEvents, MonthEvents, DayEvents, HourEvents
import pytz

from omaha.utils import get_id, is_new_install, valuedispatch, redis
from omaha.settings import DEFAULT_CHANNEL
from omaha.models import ACTIVE_USERS_DICT_CHOICES, Request, AppRequest, Os, Hw, Event, Version, Channel
from sparkle.models import SparkleVersion

__all__ = ['userid_counting', 'is_user_active']

host, port, db = settings.CACHES['statistics']['LOCATION'].split(':')
setup_redis('default', host, port, db=db)


def userid_counting(userid, apps_list, platform, now=None):
    id = get_id(userid)
    mark_event('request', id, now=now)
    list(map(partial(add_app_statistics, id, platform, now=now), apps_list or []))


def add_app_statistics(userid, platform, app, now=None):
    mark = partial(mark_event, now=now)
    if not now:
        now = timezone.now()
    appid = app.get('appid')
    version = app.get('version')
    channel = app.get('tag') or DEFAULT_CHANNEL

    if is_new_install(appid, userid):
        mark('new_install:%s' % appid, userid)
        mark('new_install:{}:{}'.format(appid, platform), userid)
        redis.setbit("known_users:%s" % appid, userid, 1)
    elif userid not in MonthEvents('new_install:{}:{}'.format(appid, platform), year=now.year, month=now.month):
        mark('request:%s' % appid, userid)
        mark('request:{}:{}'.format(appid, platform), userid)

    mark('request:{}:{}'.format(appid, version), userid)
    mark('request:{}:{}'.format(appid, channel), userid)
    mark('request:{}:{}:{}'.format(appid, platform, version), userid)

def update_live_statistics(userid, apps_list, platform, now=None):
    id = get_id(userid)
    list(map(partial(add_app_live_statistics, id, platform, now=now), apps_list or []))


def add_app_live_statistics(userid, platform, app, now=None):
    mark = partial(mark_event, now=now, track_hourly=True)
    unmark = partial(unmark_event, track_hourly=True)
    appid = app.get('appid')
    version = app.get('version')
    events = app.findall('event')
    nextversion = app.get('nextversion')

    install_event = filter(lambda x: x.get('eventtype') == '2', events)
    if install_event and install_event[0].get('eventresult') == '1':
        mark('online:{}:{}'.format(appid, nextversion), userid)
        mark('online:{}:{}:{}'.format(appid, platform, nextversion), userid)
        return

    update_event = filter(lambda x: x.get('eventtype') == '3', events)
    if update_event and update_event[0].get('eventresult') == '1':
        unmark('online:{}:{}'.format(appid, version), userid)               # necessary for
        unmark('online:{}:{}:{}'.format(appid, platform, version), userid)  # 1 hour interval
        mark('online:{}:{}'.format(appid, nextversion), userid)
        mark('online:{}:{}:{}'.format(appid, platform, nextversion), userid)
        return

    # updatecheck handling
    if version:
        mark('online:{}:{}'.format(appid, version), userid)
        mark('online:{}:{}:{}'.format(appid, platform, version), userid)

def get_users_statistics_months(app_id, platform=None, year=None, start=1, end=12):
    now = timezone.now()
    if not year:
        year = now.year

    if platform:
        install_event_name = 'new_install:{}:{}'.format(app_id, platform)
        update_event_name = 'request:{}:{}'.format(app_id, platform)
    else:
        install_event_name = 'new_install:%s' % app_id
        update_event_name = 'request:%s' % app_id

    installs_by_month = []
    updates_by_month = []
    for m in range(start, end + 1):
        installs_by_month.append(MonthEvents(install_event_name, year, m))
        updates_by_month.append(MonthEvents(update_event_name, year, m))
    installs_data = [(datetime(year, start + i, 1).strftime("%Y-%m"), len(e)) for i, e in enumerate(installs_by_month)]
    updates_data = [(datetime(year, start + i, 1).strftime("%Y-%m"), len(e)) for i, e in enumerate(updates_by_month)]
    return dict(new=installs_data, updates=updates_data)


def get_users_statistics_weeks(app_id=None):
    now = timezone.now()
    event_name = 'request:%s' % app_id if app_id else 'request'
    year = now.year
    current_week = now.isocalendar()[1]
    previous_week = (now - timedelta(weeks=1)).isocalendar()[1]
    yesterday = now - timedelta(days=1)
    data = [
        ('Previous week', len(WeekEvents(event_name, year, previous_week))),
        ('Current week', len(WeekEvents(event_name, year, current_week))),
        ('Yesterday', len(DayEvents(event_name, year, yesterday.month, yesterday.day))),
        ('Today', len(DayEvents(event_name, year, now.month, now.day))),
    ]
    return data


def get_channel_statistics(app_id, date=None):
    if not date:
        date = timezone.now()

    event_name = 'request:{}:{}'
    channels = [c.name for c in Channel.objects.all()]
    data = [(channel, len(MonthEvents(event_name.format(app_id, channel), date.year, date.month))) for channel in channels]
    data = filter(lambda x: x[1], data)
    return data


def get_users_versions_by_platform(app_id, platform, date):
    if platform == 'win':
        versions = [str(v.version) for v in Version.objects.filter_by_enabled(app__id=app_id)]
    else:
        versions = [str(v.short_version) for v in SparkleVersion.objects.filter_by_enabled(app__id=app_id)]
    event_name = 'request:{}:{}:{}'
    data = [(v, len(MonthEvents(event_name.format(app_id, platform, v), date.year, date.month))) for v in versions]
    return data


def get_users_versions(app_id, date=None):
    if not date:
        date = timezone.now()

    win_data = get_users_versions_by_platform(app_id, 'win', date)
    win_data = filter(lambda x: x[1], win_data)

    mac_data = get_users_versions_by_platform(app_id, 'mac', date)
    mac_data = filter(lambda x: x[1], mac_data)

    data = dict(win=dict(win_data), mac=dict(mac_data))

    return data



def get_versions_data_by_platform(app_id, end, n_hours, versions, platform, tz='UTC'):
    tzinfo = pytz.timezone(tz)
    start = end - timezone.timedelta(hours=n_hours)
    event_name = "online:{}:{}:{}"

    hours = [datetime(start.year, start.month, start.day, start.hour, tzinfo=pytz.UTC)
             + timezone.timedelta(hours=x) for x in range(1, n_hours + 1)]

    data = [(v, [[hour.astimezone(tzinfo).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
                  len(HourEvents.from_date(event_name.format(app_id, platform, v), hour))]
                 for hour in hours])
            for v in versions]
    data = filter(lambda version_data: sum([data[1] for data in version_data[1]]), data)
    return dict(data)


def get_users_live_versions(app_id, start, end, tz='UTC'):
    win_versions = [str(v.version) for v in Version.objects.filter_by_enabled(app__id=app_id)]
    mac_versions = [str(v.short_version) for v in SparkleVersion.objects.filter_by_enabled(app__id=app_id)]

    tmp_hours = divmod((end - start).total_seconds(), 60*60)
    n_hours = tmp_hours[0]+1
    n_hours = int(n_hours)

    win_data = get_versions_data_by_platform(app_id, end, n_hours, win_versions, 'win', tz=tz)
    mac_data = get_versions_data_by_platform(app_id, end, n_hours, mac_versions, 'mac', tz=tz)

    data = dict(win=win_data, mac=mac_data)

    return data


@valuedispatch
def is_user_active(period, userid):
    return False


@is_user_active.register(ACTIVE_USERS_DICT_CHOICES['all'])
def _(period, userid):
    return True


@is_user_active.register(ACTIVE_USERS_DICT_CHOICES['week'])
def _(period, userid):
    return get_id(userid) in WeekEvents.from_date('request', timezone.now())


@is_user_active.register(ACTIVE_USERS_DICT_CHOICES['month'])
def _(period, userid):
    return get_id(userid) in MonthEvents.from_date('request', timezone.now())


def get_kwargs_for_model(cls, obj, exclude=None):
    exclude = exclude or []
    fields = [(field.name, field.to_python) for field in cls._meta.fields if field.name not in exclude]
    kwargs = dict([(i, convert(obj.get(i))) for (i, convert) in fields])
    return kwargs


def parse_os(os):
    kwargs = get_kwargs_for_model(Os, os, exclude=['id'])
    obj, flag = Os.objects.get_or_create(**kwargs)
    return obj


def parse_hw(hw):
    kwargs = get_kwargs_for_model(Hw, hw, exclude=['id'])
    obj, flag = Hw.objects.get_or_create(**kwargs)
    return obj


def parse_req(request, ip=None):
    kwargs = get_kwargs_for_model(Request, request, exclude=['os', 'hw', 'created', 'id'])
    kwargs['ip'] = ip
    return Request(**kwargs)


def parse_apps(apps, request):
    app_list = []
    for app in apps:
        events = app.findall('event')

        if not events:
            continue

        kwargs = get_kwargs_for_model(AppRequest, app, exclude=['request', 'version', 'nextversion', 'id'])
        kwargs['version'] = app.get('version') or None
        kwargs['nextversion'] = app.get('nextversion') or None
        app_req = AppRequest.objects.create(request=request, **kwargs)
        event_list = parse_events(events)
        app_req.events.add(*event_list)
        app_list.append(app_req)
    return app_list


def parse_events(events):
    res = []
    for event in events:
        kwargs = get_kwargs_for_model(Event, event)
        res.append(Event.objects.create(**kwargs))
    return res


@transaction.atomic
def collect_statistics(request, ip=None):
    userid = request.get('userid')
    apps = request.findall('app')

    if userid:
        userid_counting(userid, apps, request.os.get('platform'))
        update_live_statistics(userid, apps, request.os.get('platform'))

    if not filter(lambda app: bool(app.findall('event')), apps):
        return

    req = parse_req(request, ip)
    req.os = parse_os(request.os)
    req.hw = parse_hw(request.hw) if request.get('hw') else None
    req.save()

    parse_apps(apps, req)



1			# coding: utf8
2
3			"""
4			This software is licensed under the Apache 2 license, quoted below.
5
6			Copyright 2014 Crystalnix Limited
7
8			Licensed under the Apache License, Version 2.0 (the "License"); you may not
9			use this file except in compliance with the License. You may obtain a copy of
10			the License at
11
12			http://www.apache.org/licenses/LICENSE-2.0
13
14			Unless required by applicable law or agreed to in writing, software
15			distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16			WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
17			License for the specific language governing permissions and limitations under
18			the License.
19			"""
20
21			from builtins import range
22
23			from functools import partial
24			from datetime import datetime, timedelta
25
26			from django.conf import settings
27			from django.db import transaction
28			from django.utils import timezone
29			from bitmapist import setup_redis, mark_event, unmark_event, WeekEvents, MonthEvents, DayEvents, HourEvents
30			import pytz
31
32			from omaha.utils import get_id, is_new_install, valuedispatch, redis
33			from omaha.settings import DEFAULT_CHANNEL
34			from omaha.models import ACTIVE_USERS_DICT_CHOICES, Request, AppRequest, Os, Hw, Event, Version, Channel
35			from sparkle.models import SparkleVersion
36
37			__all__ = ['userid_counting', 'is_user_active']
38
39			host, port, db = settings.CACHES['statistics']['LOCATION'].split(':')
40			setup_redis('default', host, port, db=db)
41
42
43			def userid_counting(userid, apps_list, platform, now=None):
44			id = get_id(userid)
45			mark_event('request', id, now=now)
46			list(map(partial(add_app_statistics, id, platform, now=now), apps_list or []))
47
48
49			def add_app_statistics(userid, platform, app, now=None):
50			mark = partial(mark_event, now=now)
51			if not now:
52			now = timezone.now()
53			appid = app.get('appid')
54			version = app.get('version')
55			channel = app.get('tag') or DEFAULT_CHANNEL
56
57			if is_new_install(appid, userid):
58			mark('new_install:%s' % appid, userid)
59			mark('new_install:{}:{}'.format(appid, platform), userid)
60			redis.setbit("known_users:%s" % appid, userid, 1)
61			elif userid not in MonthEvents('new_install:{}:{}'.format(appid, platform), year=now.year, month=now.month):
62			mark('request:%s' % appid, userid)
63			mark('request:{}:{}'.format(appid, platform), userid)
64
65			mark('request:{}:{}'.format(appid, version), userid)
66			mark('request:{}:{}'.format(appid, channel), userid)
67			mark('request:{}:{}:{}'.format(appid, platform, version), userid)
68
69			def update_live_statistics(userid, apps_list, platform, now=None):
70			id = get_id(userid)
71			list(map(partial(add_app_live_statistics, id, platform, now=now), apps_list or []))
72
73
74			def add_app_live_statistics(userid, platform, app, now=None):
75			mark = partial(mark_event, now=now, track_hourly=True)
76			unmark = partial(unmark_event, track_hourly=True)
77			appid = app.get('appid')
78			version = app.get('version')
79			events = app.findall('event')
80			nextversion = app.get('nextversion')
81
82			install_event = filter(lambda x: x.get('eventtype') == '2', events)
83			if install_event and install_event[0].get('eventresult') == '1':
84			mark('online:{}:{}'.format(appid, nextversion), userid)
85			mark('online:{}:{}:{}'.format(appid, platform, nextversion), userid)
86			return
87
88			update_event = filter(lambda x: x.get('eventtype') == '3', events)
89			if update_event and update_event[0].get('eventresult') == '1':
90			unmark('online:{}:{}'.format(appid, version), userid) # necessary for
91			unmark('online:{}:{}:{}'.format(appid, platform, version), userid) # 1 hour interval
92			mark('online:{}:{}'.format(appid, nextversion), userid)
93			mark('online:{}:{}:{}'.format(appid, platform, nextversion), userid)
94			return
95
96			# updatecheck handling
97			if version:
98			mark('online:{}:{}'.format(appid, version), userid)
99			mark('online:{}:{}:{}'.format(appid, platform, version), userid)
100
101			def get_users_statistics_months(app_id, platform=None, year=None, start=1, end=12):
102			now = timezone.now()
103			if not year:
104			year = now.year
105
106			if platform:
107			install_event_name = 'new_install:{}:{}'.format(app_id, platform)
108			update_event_name = 'request:{}:{}'.format(app_id, platform)
109			else:
110			install_event_name = 'new_install:%s' % app_id
111			update_event_name = 'request:%s' % app_id
112
113			installs_by_month = []
114			updates_by_month = []
115			for m in range(start, end + 1):
116			installs_by_month.append(MonthEvents(install_event_name, year, m))
117			updates_by_month.append(MonthEvents(update_event_name, year, m))
118			installs_data = [(datetime(year, start + i, 1).strftime("%Y-%m"), len(e)) for i, e in enumerate(installs_by_month)]
119			updates_data = [(datetime(year, start + i, 1).strftime("%Y-%m"), len(e)) for i, e in enumerate(updates_by_month)]
120			return dict(new=installs_data, updates=updates_data)
121
122
123			def get_users_statistics_weeks(app_id=None):
124			now = timezone.now()
125			event_name = 'request:%s' % app_id if app_id else 'request'
126			year = now.year
127			current_week = now.isocalendar()[1]
128			previous_week = (now - timedelta(weeks=1)).isocalendar()[1]
129			yesterday = now - timedelta(days=1)
130			data = [
131			('Previous week', len(WeekEvents(event_name, year, previous_week))),
132			('Current week', len(WeekEvents(event_name, year, current_week))),
133			('Yesterday', len(DayEvents(event_name, year, yesterday.month, yesterday.day))),
134			('Today', len(DayEvents(event_name, year, now.month, now.day))),
135			]
136			return data
137
138
139			def get_channel_statistics(app_id, date=None):
140			if not date:
141			date = timezone.now()
142
143			event_name = 'request:{}:{}'
144			channels = [c.name for c in Channel.objects.all()]
145			data = [(channel, len(MonthEvents(event_name.format(app_id, channel), date.year, date.month))) for channel in channels]
146			data = filter(lambda x: x[1], data)
147			return data
148
149
150			def get_users_versions_by_platform(app_id, platform, date):
151			if platform == 'win':
152			versions = [str(v.version) for v in Version.objects.filter_by_enabled(app__id=app_id)]
153			else:
154			versions = [str(v.short_version) for v in SparkleVersion.objects.filter_by_enabled(app__id=app_id)]
155			event_name = 'request:{}:{}:{}'
156			data = [(v, len(MonthEvents(event_name.format(app_id, platform, v), date.year, date.month))) for v in versions]
157			return data
158
159
160			def get_users_versions(app_id, date=None):
161			if not date:
162			date = timezone.now()
163
164			win_data = get_users_versions_by_platform(app_id, 'win', date)
165			win_data = filter(lambda x: x[1], win_data)
166
167			mac_data = get_users_versions_by_platform(app_id, 'mac', date)
168			mac_data = filter(lambda x: x[1], mac_data)
169
170			data = dict(win=dict(win_data), mac=dict(mac_data))
171
172			return data
173
174
175
176			def get_versions_data_by_platform(app_id, end, n_hours, versions, platform, tz='UTC'):
177			tzinfo = pytz.timezone(tz)
178			start = end - timezone.timedelta(hours=n_hours)
179			event_name = "online:{}:{}:{}"
180
181			hours = [datetime(start.year, start.month, start.day, start.hour, tzinfo=pytz.UTC)
182			+ timezone.timedelta(hours=x) for x in range(1, n_hours + 1)]
183
184			data = [(v, [[hour.astimezone(tzinfo).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
185			len(HourEvents.from_date(event_name.format(app_id, platform, v), hour))]
186			for hour in hours])
187			for v in versions]
188			data = filter(lambda version_data: sum([data[1] for data in version_data[1]]), data)
189			return dict(data)
190
191
192			def get_users_live_versions(app_id, start, end, tz='UTC'):
193			win_versions = [str(v.version) for v in Version.objects.filter_by_enabled(app__id=app_id)]
194			mac_versions = [str(v.short_version) for v in SparkleVersion.objects.filter_by_enabled(app__id=app_id)]
195
196			tmp_hours = divmod((end - start).total_seconds(), 60*60)
197			n_hours = tmp_hours[0]+1
198			n_hours = int(n_hours)
199
200			win_data = get_versions_data_by_platform(app_id, end, n_hours, win_versions, 'win', tz=tz)
201			mac_data = get_versions_data_by_platform(app_id, end, n_hours, mac_versions, 'mac', tz=tz)
202
203			data = dict(win=win_data, mac=mac_data)
204
205			return data
206
207
208			@valuedispatch
209			def is_user_active(period, userid):
210			return False
211
212
213			@is_user_active.register(ACTIVE_USERS_DICT_CHOICES['all'])
214			def _(period, userid):
215			return True
216
217
218			@is_user_active.register(ACTIVE_USERS_DICT_CHOICES['week'])
219			def _(period, userid):
220			return get_id(userid) in WeekEvents.from_date('request', timezone.now())
221
222
223			@is_user_active.register(ACTIVE_USERS_DICT_CHOICES['month'])
224			def _(period, userid):
225			return get_id(userid) in MonthEvents.from_date('request', timezone.now())
226
227
228			def get_kwargs_for_model(cls, obj, exclude=None):
229			exclude = exclude or []
230			fields = [(field.name, field.to_python) for field in cls._meta.fields if field.name not in exclude]
231			kwargs = dict([(i, convert(obj.get(i))) for (i, convert) in fields])
232			return kwargs
233
234
235			def parse_os(os):
236			kwargs = get_kwargs_for_model(Os, os, exclude=['id'])
237			obj, flag = Os.objects.get_or_create(**kwargs)
238			return obj
239
240
241			def parse_hw(hw):
242			kwargs = get_kwargs_for_model(Hw, hw, exclude=['id'])
243			obj, flag = Hw.objects.get_or_create(**kwargs)
244			return obj
245
246
247			def parse_req(request, ip=None):
248			kwargs = get_kwargs_for_model(Request, request, exclude=['os', 'hw', 'created', 'id'])
249			kwargs['ip'] = ip
250			return Request(**kwargs)
251
252
253			def parse_apps(apps, request):
254			app_list = []
255			for app in apps:
256			events = app.findall('event')
257
258			if not events:
259			continue
260
261			kwargs = get_kwargs_for_model(AppRequest, app, exclude=['request', 'version', 'nextversion', 'id'])
262			kwargs['version'] = app.get('version') or None
263			kwargs['nextversion'] = app.get('nextversion') or None
264			app_req = AppRequest.objects.create(request=request, **kwargs)
265			event_list = parse_events(events)
266			app_req.events.add(*event_list)
267			app_list.append(app_req)
268			return app_list
269
270
271			def parse_events(events):
272			res = []
273			for event in events:
274			kwargs = get_kwargs_for_model(Event, event)
275			res.append(Event.objects.create(**kwargs))
276			return res
277
278
279			@transaction.atomic
280			def collect_statistics(request, ip=None):
281			userid = request.get('userid')
282			apps = request.findall('app')
283
284			if userid:
285			userid_counting(userid, apps, request.os.get('platform'))
286			update_live_statistics(userid, apps, request.os.get('platform'))
287
288			if not filter(lambda app: bool(app.findall('event')), apps):
289			return
290
291			req = parse_req(request, ip)
292			req.os = parse_os(request.os)
293			req.hw = parse_hw(request.hw) if request.get('hw') else None
294			req.save()
295
296			parse_apps(apps, req)
297
298

Crystalnix / omaha-server

Push — master ( 5fa8a3...254aa9 )

add_app_live_statistics() D

Complexity

Size

Duplication

Duplication Side-by-Side

Filter issues like