get_channel_statistics() - Code Metrics - Inspection of "Bump version: 0.3.2 → 0.3.3" - Crystalnix/omaha-server - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( 67eae9...f0f1ec )

unknown

created 2016-04-07 06:56 UTC

get_channel_statistics() B

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Metric	Value
dl	0
loc	9
rs	8.5454
cc	5

# coding: utf8

"""
This software is licensed under the Apache 2 license, quoted below.

Copyright 2014 Crystalnix Limited

Licensed under the Apache License, Version 2.0 (the "License"); you may not
use this file except in compliance with the License. You may obtain a copy of
the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations under
the License.
"""

from builtins import range

from functools import partial
from datetime import datetime, timedelta

from django.conf import settings
from django.db import transaction
from django.utils import timezone
from bitmapist import setup_redis, mark_event, unmark_event, WeekEvents, MonthEvents, DayEvents, HourEvents
import pytz

from omaha.utils import get_id, is_new_install, valuedispatch, redis
from omaha.settings import DEFAULT_CHANNEL
from omaha.models import ACTIVE_USERS_DICT_CHOICES, Request, AppRequest, Os, Hw, Event, Version, Channel
from sparkle.models import SparkleVersion

__all__ = ['userid_counting', 'is_user_active']

host, port, db = settings.CACHES['statistics']['LOCATION'].split(':')
setup_redis('default', host, port, db=db)


def userid_counting(userid, apps_list, platform, now=None):
    id = get_id(userid)
    mark_event('request', id, now=now)
    list(map(partial(add_app_statistics, id, platform, now=now), apps_list or []))


def add_app_statistics(userid, platform, app, now=None):
    mark = partial(mark_event, now=now)
    if not now:
        now = timezone.now()
    appid = app.get('appid')
    version = app.get('version')
    channel = app.get('tag') or DEFAULT_CHANNEL

    if is_new_install(appid, userid):
        mark('new_install:%s' % appid, userid)
        mark('new_install:{}:{}'.format(appid, platform), userid)
        redis.setbit("known_users:%s" % appid, userid, 1)
    elif userid not in MonthEvents('new_install:{}:{}'.format(appid, platform), year=now.year, month=now.month):
        mark('request:%s' % appid, userid)
        mark('request:{}:{}'.format(appid, platform), userid)

    mark('request:{}:{}'.format(appid, version), userid)
    mark('request:{}:{}'.format(appid, channel), userid)
    mark('request:{}:{}:{}'.format(appid, platform, version), userid)

def update_live_statistics(userid, apps_list, platform, now=None):
    id = get_id(userid)
    list(map(partial(add_app_live_statistics, id, platform, now=now), apps_list or []))


def add_app_live_statistics(userid, platform, app, now=None):
    mark = partial(mark_event, now=now, track_hourly=True)
    unmark = partial(unmark_event, track_hourly=True)
    appid = app.get('appid')
    version = app.get('version')
    events = app.findall('event')
    nextversion = app.get('nextversion')

    install_event = filter(lambda x: x.get('eventtype') == '2', events)
    if install_event and install_event[0].get('eventresult') == '1':
        mark('online:{}:{}'.format(appid, nextversion), userid)
        mark('online:{}:{}:{}'.format(appid, platform, nextversion), userid)
        return

    update_event = filter(lambda x: x.get('eventtype') == '3', events)
    if update_event and update_event[0].get('eventresult') == '1':
        unmark('online:{}:{}'.format(appid, version), userid)               # necessary for
        unmark('online:{}:{}:{}'.format(appid, platform, version), userid)  # 1 hour interval
        mark('online:{}:{}'.format(appid, nextversion), userid)
        mark('online:{}:{}:{}'.format(appid, platform, nextversion), userid)
        return

    uninstall_event = filter(lambda x: x.get('eventtype') == '4', events)
    if uninstall_event and uninstall_event[0].get('eventresult') == '1':
        unmark('online:{}:{}'.format(appid, version), userid)               # necessary for
        unmark('online:{}:{}:{}'.format(appid, platform, version), userid)  # 1 hour interval
        return

    # updatecheck handling
    if version:
        mark('online:{}:{}'.format(appid, version), userid)
        mark('online:{}:{}:{}'.format(appid, platform, version), userid)

def get_users_statistics_months(app_id, platform=None, year=None, start=1, end=12):
    now = timezone.now()
    if not year:
        year = now.year

    if platform:
        install_event_name = 'new_install:{}:{}'.format(app_id, platform)
        update_event_name = 'request:{}:{}'.format(app_id, platform)
    else:
        install_event_name = 'new_install:%s' % app_id
        update_event_name = 'request:%s' % app_id

    installs_by_month = []
    updates_by_month = []
    for m in range(start, end + 1):
        installs_by_month.append(MonthEvents(install_event_name, year, m))
        updates_by_month.append(MonthEvents(update_event_name, year, m))
    installs_data = [(datetime(year, start + i, 1).strftime("%Y-%m"), len(e)) for i, e in enumerate(installs_by_month)]
    updates_data = [(datetime(year, start + i, 1).strftime("%Y-%m"), len(e)) for i, e in enumerate(updates_by_month)]
    return dict(new=installs_data, updates=updates_data)


def get_users_statistics_weeks(app_id=None):
    now = timezone.now()
    event_name = 'request:%s' % app_id if app_id else 'request'
    year = now.year
    current_week = now.isocalendar()[1]
    previous_week = (now - timedelta(weeks=1)).isocalendar()[1]
    yesterday = now - timedelta(days=1)
    data = [
        ('Previous week', len(WeekEvents(event_name, year, previous_week))),
        ('Current week', len(WeekEvents(event_name, year, current_week))),
        ('Yesterday', len(DayEvents(event_name, year, yesterday.month, yesterday.day))),
        ('Today', len(DayEvents(event_name, year, now.month, now.day))),
    ]
    return data


def get_channel_statistics(app_id, date=None):
    if not date:
        date = timezone.now()

    event_name = 'request:{}:{}'
    channels = [c.name for c in Channel.objects.all()]
    data = [(channel, len(MonthEvents(event_name.format(app_id, channel), date.year, date.month))) for channel in channels]
    data = filter(lambda x: x[1], data)
    return data


def get_users_versions_by_platform(app_id, platform, date):
    if platform == 'win':
        versions = [str(v.version) for v in Version.objects.filter_by_enabled(app__id=app_id)]
    else:
        versions = [str(v.short_version) for v in SparkleVersion.objects.filter_by_enabled(app__id=app_id)]
    event_name = 'request:{}:{}:{}'
    data = [(v, len(MonthEvents(event_name.format(app_id, platform, v), date.year, date.month))) for v in versions]
    return data


def get_users_versions(app_id, date=None):
    if not date:
        date = timezone.now()

    win_data = get_users_versions_by_platform(app_id, 'win', date)
    win_data = filter(lambda x: x[1], win_data)

    mac_data = get_users_versions_by_platform(app_id, 'mac', date)
    mac_data = filter(lambda x: x[1], mac_data)

    data = dict(win=dict(win_data), mac=dict(mac_data))

    return data



def get_versions_data_by_platform(app_id, end, n_hours, versions, platform, tz='UTC'):
    tzinfo = pytz.timezone(tz)
    start = end - timezone.timedelta(hours=n_hours)
    event_name = "online:{}:{}:{}"

    hours = [datetime(start.year, start.month, start.day, start.hour, tzinfo=pytz.UTC)
             + timezone.timedelta(hours=x) for x in range(1, n_hours + 1)]

    data = [(v, [[hour.astimezone(tzinfo).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
                  len(HourEvents.from_date(event_name.format(app_id, platform, v), hour))]
                 for hour in hours])
            for v in versions]
    data = filter(lambda version_data: sum([data[1] for data in version_data[1]]), data)
    return dict(data)


def get_users_live_versions(app_id, start, end, tz='UTC'):
    win_versions = [str(v.version) for v in Version.objects.filter_by_enabled(app__id=app_id)]
    mac_versions = [str(v.short_version) for v in SparkleVersion.objects.filter_by_enabled(app__id=app_id)]

    tmp_hours = divmod((end - start).total_seconds(), 60*60)
    n_hours = tmp_hours[0]+1
    n_hours = int(n_hours)

    win_data = get_versions_data_by_platform(app_id, end, n_hours, win_versions, 'win', tz=tz)
    mac_data = get_versions_data_by_platform(app_id, end, n_hours, mac_versions, 'mac', tz=tz)

    data = dict(win=win_data, mac=mac_data)

    return data


@valuedispatch
def is_user_active(period, userid):
    return False


@is_user_active.register(ACTIVE_USERS_DICT_CHOICES['all'])
def _(period, userid):
    return True


@is_user_active.register(ACTIVE_USERS_DICT_CHOICES['week'])
def _(period, userid):
    return get_id(userid) in WeekEvents.from_date('request', timezone.now())


@is_user_active.register(ACTIVE_USERS_DICT_CHOICES['month'])
def _(period, userid):
    return get_id(userid) in MonthEvents.from_date('request', timezone.now())


def get_kwargs_for_model(cls, obj, exclude=None):
    exclude = exclude or []
    fields = [(field.name, field.to_python) for field in cls._meta.fields if field.name not in exclude]
    kwargs = dict([(i, convert(obj.get(i))) for (i, convert) in fields])
    return kwargs


def parse_os(os):
    kwargs = get_kwargs_for_model(Os, os, exclude=['id'])
    obj, flag = Os.objects.get_or_create(**kwargs)
    return obj


def parse_hw(hw):
    kwargs = get_kwargs_for_model(Hw, hw, exclude=['id'])
    obj, flag = Hw.objects.get_or_create(**kwargs)
    return obj


def parse_req(request, ip=None):
    kwargs = get_kwargs_for_model(Request, request, exclude=['os', 'hw', 'created', 'id'])
    kwargs['ip'] = ip
    return Request(**kwargs)


def parse_apps(apps, request):
    app_list = []
    for app in apps:
        events = app.findall('event')

        if not events:
            continue

        kwargs = get_kwargs_for_model(AppRequest, app, exclude=['request', 'version', 'nextversion', 'id'])
        kwargs['version'] = app.get('version') or None
        kwargs['nextversion'] = app.get('nextversion') or None
        app_req = AppRequest.objects.create(request=request, **kwargs)
        event_list = parse_events(events)
        app_req.events.add(*event_list)
        app_list.append(app_req)
    return app_list


def parse_events(events):
    res = []
    for event in events:
        kwargs = get_kwargs_for_model(Event, event)
        res.append(Event.objects.create(**kwargs))
    return res


@transaction.atomic
def collect_statistics(request, ip=None):
    userid = request.get('userid')
    apps = request.findall('app')

    if userid:
        userid_counting(userid, apps, request.os.get('platform'))
        update_live_statistics(userid, apps, request.os.get('platform'))

    if not filter(lambda app: bool(app.findall('event')), apps):
        return

    req = parse_req(request, ip)
    req.os = parse_os(request.os)
    req.hw = parse_hw(request.hw) if request.get('hw') else None
    req.save()

    parse_apps(apps, req)



1			# coding: utf8
2
3			"""
4			This software is licensed under the Apache 2 license, quoted below.
5
6			Copyright 2014 Crystalnix Limited
7
8			Licensed under the Apache License, Version 2.0 (the "License"); you may not
9			use this file except in compliance with the License. You may obtain a copy of
10			the License at
11
12			http://www.apache.org/licenses/LICENSE-2.0
13
14			Unless required by applicable law or agreed to in writing, software
15			distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16			WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
17			License for the specific language governing permissions and limitations under
18			the License.
19			"""
20
21			from builtins import range
22
23			from functools import partial
24			from datetime import datetime, timedelta
25
26			from django.conf import settings
27			from django.db import transaction
28			from django.utils import timezone
29			from bitmapist import setup_redis, mark_event, unmark_event, WeekEvents, MonthEvents, DayEvents, HourEvents
30			import pytz
31
32			from omaha.utils import get_id, is_new_install, valuedispatch, redis
33			from omaha.settings import DEFAULT_CHANNEL
34			from omaha.models import ACTIVE_USERS_DICT_CHOICES, Request, AppRequest, Os, Hw, Event, Version, Channel
35			from sparkle.models import SparkleVersion
36
37			__all__ = ['userid_counting', 'is_user_active']
38
39			host, port, db = settings.CACHES['statistics']['LOCATION'].split(':')
40			setup_redis('default', host, port, db=db)
41
42
43			def userid_counting(userid, apps_list, platform, now=None):
44			id = get_id(userid)
45			mark_event('request', id, now=now)
46			list(map(partial(add_app_statistics, id, platform, now=now), apps_list or []))
47
48
49			def add_app_statistics(userid, platform, app, now=None):
50			mark = partial(mark_event, now=now)
51			if not now:
52			now = timezone.now()
53			appid = app.get('appid')
54			version = app.get('version')
55			channel = app.get('tag') or DEFAULT_CHANNEL
56
57			if is_new_install(appid, userid):
58			mark('new_install:%s' % appid, userid)
59			mark('new_install:{}:{}'.format(appid, platform), userid)
60			redis.setbit("known_users:%s" % appid, userid, 1)
61			elif userid not in MonthEvents('new_install:{}:{}'.format(appid, platform), year=now.year, month=now.month):
62			mark('request:%s' % appid, userid)
63			mark('request:{}:{}'.format(appid, platform), userid)
64
65			mark('request:{}:{}'.format(appid, version), userid)
66			mark('request:{}:{}'.format(appid, channel), userid)
67			mark('request:{}:{}:{}'.format(appid, platform, version), userid)
68
69			def update_live_statistics(userid, apps_list, platform, now=None):
70			id = get_id(userid)
71			list(map(partial(add_app_live_statistics, id, platform, now=now), apps_list or []))
72
73
74			def add_app_live_statistics(userid, platform, app, now=None):
75			mark = partial(mark_event, now=now, track_hourly=True)
76			unmark = partial(unmark_event, track_hourly=True)
77			appid = app.get('appid')
78			version = app.get('version')
79			events = app.findall('event')
80			nextversion = app.get('nextversion')
81
82			install_event = filter(lambda x: x.get('eventtype') == '2', events)
83			if install_event and install_event[0].get('eventresult') == '1':
84			mark('online:{}:{}'.format(appid, nextversion), userid)
85			mark('online:{}:{}:{}'.format(appid, platform, nextversion), userid)
86			return
87
88			update_event = filter(lambda x: x.get('eventtype') == '3', events)
89			if update_event and update_event[0].get('eventresult') == '1':
90			unmark('online:{}:{}'.format(appid, version), userid) # necessary for
91			unmark('online:{}:{}:{}'.format(appid, platform, version), userid) # 1 hour interval
92			mark('online:{}:{}'.format(appid, nextversion), userid)
93			mark('online:{}:{}:{}'.format(appid, platform, nextversion), userid)
94			return
95
96			uninstall_event = filter(lambda x: x.get('eventtype') == '4', events)
97			if uninstall_event and uninstall_event[0].get('eventresult') == '1':
98			unmark('online:{}:{}'.format(appid, version), userid) # necessary for
99			unmark('online:{}:{}:{}'.format(appid, platform, version), userid) # 1 hour interval
100			return
101
102			# updatecheck handling
103			if version:
104			mark('online:{}:{}'.format(appid, version), userid)
105			mark('online:{}:{}:{}'.format(appid, platform, version), userid)
106
107			def get_users_statistics_months(app_id, platform=None, year=None, start=1, end=12):
108			now = timezone.now()
109			if not year:
110			year = now.year
111
112			if platform:
113			install_event_name = 'new_install:{}:{}'.format(app_id, platform)
114			update_event_name = 'request:{}:{}'.format(app_id, platform)
115			else:
116			install_event_name = 'new_install:%s' % app_id
117			update_event_name = 'request:%s' % app_id
118
119			installs_by_month = []
120			updates_by_month = []
121			for m in range(start, end + 1):
122			installs_by_month.append(MonthEvents(install_event_name, year, m))
123			updates_by_month.append(MonthEvents(update_event_name, year, m))
124			installs_data = [(datetime(year, start + i, 1).strftime("%Y-%m"), len(e)) for i, e in enumerate(installs_by_month)]
125			updates_data = [(datetime(year, start + i, 1).strftime("%Y-%m"), len(e)) for i, e in enumerate(updates_by_month)]
126			return dict(new=installs_data, updates=updates_data)
127
128
129			def get_users_statistics_weeks(app_id=None):
130			now = timezone.now()
131			event_name = 'request:%s' % app_id if app_id else 'request'
132			year = now.year
133			current_week = now.isocalendar()[1]
134			previous_week = (now - timedelta(weeks=1)).isocalendar()[1]
135			yesterday = now - timedelta(days=1)
136			data = [
137			('Previous week', len(WeekEvents(event_name, year, previous_week))),
138			('Current week', len(WeekEvents(event_name, year, current_week))),
139			('Yesterday', len(DayEvents(event_name, year, yesterday.month, yesterday.day))),
140			('Today', len(DayEvents(event_name, year, now.month, now.day))),
141			]
142			return data
143
144
145			def get_channel_statistics(app_id, date=None):
146			if not date:
147			date = timezone.now()
148
149			event_name = 'request:{}:{}'
150			channels = [c.name for c in Channel.objects.all()]
151			data = [(channel, len(MonthEvents(event_name.format(app_id, channel), date.year, date.month))) for channel in channels]
152			data = filter(lambda x: x[1], data)
153			return data
154
155
156			def get_users_versions_by_platform(app_id, platform, date):
157			if platform == 'win':
158			versions = [str(v.version) for v in Version.objects.filter_by_enabled(app__id=app_id)]
159			else:
160			versions = [str(v.short_version) for v in SparkleVersion.objects.filter_by_enabled(app__id=app_id)]
161			event_name = 'request:{}:{}:{}'
162			data = [(v, len(MonthEvents(event_name.format(app_id, platform, v), date.year, date.month))) for v in versions]
163			return data
164
165
166			def get_users_versions(app_id, date=None):
167			if not date:
168			date = timezone.now()
169
170			win_data = get_users_versions_by_platform(app_id, 'win', date)
171			win_data = filter(lambda x: x[1], win_data)
172
173			mac_data = get_users_versions_by_platform(app_id, 'mac', date)
174			mac_data = filter(lambda x: x[1], mac_data)
175
176			data = dict(win=dict(win_data), mac=dict(mac_data))
177
178			return data
179
180
181
182			def get_versions_data_by_platform(app_id, end, n_hours, versions, platform, tz='UTC'):
183			tzinfo = pytz.timezone(tz)
184			start = end - timezone.timedelta(hours=n_hours)
185			event_name = "online:{}:{}:{}"
186
187			hours = [datetime(start.year, start.month, start.day, start.hour, tzinfo=pytz.UTC)
188			+ timezone.timedelta(hours=x) for x in range(1, n_hours + 1)]
189
190			data = [(v, [[hour.astimezone(tzinfo).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
191			len(HourEvents.from_date(event_name.format(app_id, platform, v), hour))]
192			for hour in hours])
193			for v in versions]
194			data = filter(lambda version_data: sum([data[1] for data in version_data[1]]), data)
195			return dict(data)
196
197
198			def get_users_live_versions(app_id, start, end, tz='UTC'):
199			win_versions = [str(v.version) for v in Version.objects.filter_by_enabled(app__id=app_id)]
200			mac_versions = [str(v.short_version) for v in SparkleVersion.objects.filter_by_enabled(app__id=app_id)]
201
202			tmp_hours = divmod((end - start).total_seconds(), 60*60)
203			n_hours = tmp_hours[0]+1
204			n_hours = int(n_hours)
205
206			win_data = get_versions_data_by_platform(app_id, end, n_hours, win_versions, 'win', tz=tz)
207			mac_data = get_versions_data_by_platform(app_id, end, n_hours, mac_versions, 'mac', tz=tz)
208
209			data = dict(win=win_data, mac=mac_data)
210
211			return data
212
213
214			@valuedispatch
215			def is_user_active(period, userid):
216			return False
217
218
219			@is_user_active.register(ACTIVE_USERS_DICT_CHOICES['all'])
220			def _(period, userid):
221			return True
222
223
224			@is_user_active.register(ACTIVE_USERS_DICT_CHOICES['week'])
225			def _(period, userid):
226			return get_id(userid) in WeekEvents.from_date('request', timezone.now())
227
228
229			@is_user_active.register(ACTIVE_USERS_DICT_CHOICES['month'])
230			def _(period, userid):
231			return get_id(userid) in MonthEvents.from_date('request', timezone.now())
232
233
234			def get_kwargs_for_model(cls, obj, exclude=None):
235			exclude = exclude or []
236			fields = [(field.name, field.to_python) for field in cls._meta.fields if field.name not in exclude]
237			kwargs = dict([(i, convert(obj.get(i))) for (i, convert) in fields])
238			return kwargs
239
240
241			def parse_os(os):
242			kwargs = get_kwargs_for_model(Os, os, exclude=['id'])
243			obj, flag = Os.objects.get_or_create(**kwargs)
244			return obj
245
246
247			def parse_hw(hw):
248			kwargs = get_kwargs_for_model(Hw, hw, exclude=['id'])
249			obj, flag = Hw.objects.get_or_create(**kwargs)
250			return obj
251
252
253			def parse_req(request, ip=None):
254			kwargs = get_kwargs_for_model(Request, request, exclude=['os', 'hw', 'created', 'id'])
255			kwargs['ip'] = ip
256			return Request(**kwargs)
257
258
259			def parse_apps(apps, request):
260			app_list = []
261			for app in apps:
262			events = app.findall('event')
263
264			if not events:
265			continue
266
267			kwargs = get_kwargs_for_model(AppRequest, app, exclude=['request', 'version', 'nextversion', 'id'])
268			kwargs['version'] = app.get('version') or None
269			kwargs['nextversion'] = app.get('nextversion') or None
270			app_req = AppRequest.objects.create(request=request, **kwargs)
271			event_list = parse_events(events)
272			app_req.events.add(*event_list)
273			app_list.append(app_req)
274			return app_list
275
276
277			def parse_events(events):
278			res = []
279			for event in events:
280			kwargs = get_kwargs_for_model(Event, event)
281			res.append(Event.objects.create(**kwargs))
282			return res
283
284
285			@transaction.atomic
286			def collect_statistics(request, ip=None):
287			userid = request.get('userid')
288			apps = request.findall('app')
289
290			if userid:
291			userid_counting(userid, apps, request.os.get('platform'))
292			update_live_statistics(userid, apps, request.os.get('platform'))
293
294			if not filter(lambda app: bool(app.findall('event')), apps):
295			return
296
297			req = parse_req(request, ip)
298			req.os = parse_os(request.os)
299			req.hw = parse_hw(request.hw) if request.get('hw') else None
300			req.save()
301
302			parse_apps(apps, req)
303
304

Crystalnix / omaha-server

Push — master ( 67eae9...f0f1ec )

get_channel_statistics() B

Complexity

Size

Duplication

Duplication Side-by-Side

Filter issues like