collect_statistics() - Code Metrics - Inspection of "Bump version: 0.3.0 → 0.3.1" - Crystalnix/omaha-server - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( b7eb8c...589bde )

by Egor

created 2016-03-25 12:29 UTC

collect_statistics() B

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Metric	Value
dl	0
loc	18
rs	8.5454
cc	5

# coding: utf8

"""
This software is licensed under the Apache 2 license, quoted below.

Copyright 2014 Crystalnix Limited

Licensed under the Apache License, Version 2.0 (the "License"); you may not
use this file except in compliance with the License. You may obtain a copy of
the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations under
the License.
"""

from builtins import range

from functools import partial
from datetime import datetime, timedelta

from django.conf import settings
from django.db import transaction
from django.utils import timezone
from bitmapist import setup_redis, mark_event, unmark_event, WeekEvents, MonthEvents, DayEvents, HourEvents
import pytz

from omaha.utils import get_id, valuedispatch
from omaha.settings import DEFAULT_CHANNEL
from omaha.models import ACTIVE_USERS_DICT_CHOICES, Request, AppRequest, Os, Hw, Event, Version, Channel
from sparkle.models import SparkleVersion

__all__ = ['userid_counting', 'is_user_active']

host, port, db = settings.CACHES['statistics']['LOCATION'].split(':')
setup_redis('default', host, port, db=db)


def userid_counting(userid, apps_list, platform, now=None):
    id = get_id(userid)
    mark_event('request', id, now=now)
    list(map(partial(add_app_statistics, id, platform, now=now), apps_list or []))


def add_app_statistics(userid, platform, app, now=None):
    mark = partial(mark_event, now=now)
    appid = app.get('appid')
    version = app.get('version')
    channel = app.get('tag') or DEFAULT_CHANNEL
    mark('request:%s' % appid, userid)
    mark('request:{}:{}'.format(appid, version), userid)
    mark('request:{}:{}'.format(appid, platform), userid)
    mark('request:{}:{}'.format(appid, channel), userid)
    mark('request:{}:{}:{}'.format(appid, platform, version), userid)

def update_live_statistics(userid, apps_list, platform, now=None):
    id = get_id(userid)
    list(map(partial(add_app_live_statistics, id, platform, now=now), apps_list or []))


def add_app_live_statistics(userid, platform, app, now=None):
    mark = partial(mark_event, now=now, track_hourly=True)
    unmark = partial(unmark_event, track_hourly=True)
    appid = app.get('appid')
    version = app.get('version')
    events = app.findall('event')
    nextversion = app.get('nextversion')

    install_event = filter(lambda x: x.get('eventtype') == '2', events)
    if install_event and install_event[0].get('eventresult') == '1':
        mark('online:{}:{}'.format(appid, nextversion), userid)
        mark('online:{}:{}:{}'.format(appid, platform, nextversion), userid)
        return

    update_event = filter(lambda x: x.get('eventtype') == '3', events)
    if update_event and update_event[0].get('eventresult') == '1':
        unmark('online:{}:{}'.format(appid, version), userid)               # necessary for
        unmark('online:{}:{}:{}'.format(appid, platform, version), userid)  # 1 hour interval
        mark('online:{}:{}'.format(appid, nextversion), userid)
        mark('online:{}:{}:{}'.format(appid, platform, nextversion), userid)
        return

    uninstall_event = filter(lambda x: x.get('eventtype') == '4', events)
    if uninstall_event and uninstall_event[0].get('eventresult') == '1':
        unmark('online:{}:{}'.format(appid, version), userid)               # necessary for
        unmark('online:{}:{}:{}'.format(appid, platform, version), userid)  # 1 hour interval
        return

    # updatecheck handling
    if version:
        mark('online:{}:{}'.format(appid, version), userid)
        mark('online:{}:{}:{}'.format(appid, platform, version), userid)

def get_users_statistics_months(app_id=None, year=None, start=1, end=12):
    now = timezone.now()
    if not year:
        year = now.year
    event_name = 'request:%s' % app_id if app_id else 'request'

    months = []
    for m in range(start, end + 1):
        months.append(MonthEvents(event_name, year, m))
    data = [(datetime(year, start + i, 1).strftime("%Y-%m"), len(e)) for i, e in enumerate(months)]
    return data


def get_users_statistics_weeks(app_id=None):
    now = timezone.now()
    event_name = 'request:%s' % app_id if app_id else 'request'
    year = now.year
    current_week = now.isocalendar()[1]
    previous_week = (now - timedelta(weeks=1)).isocalendar()[1]
    yesterday = now - timedelta(days=1)
    data = [
        ('Previous week', len(WeekEvents(event_name, year, previous_week))),
        ('Current week', len(WeekEvents(event_name, year, current_week))),
        ('Yesterday', len(DayEvents(event_name, year, yesterday.month, yesterday.day))),
        ('Today', len(DayEvents(event_name, year, now.month, now.day))),
    ]
    return data


def get_channel_statistics(app_id):
    now = timezone.now()
    event_name = 'request:{}:{}'
    week = now.isocalendar()[1]
    channels = [c.name for c in Channel.objects.all()]
    data = [(channel, len(WeekEvents(event_name.format(app_id, channel), now.year, week))) for channel in channels]
    return data


def get_users_versions(app_id):
    now = timezone.now()
    event_name = 'request:{}:{}'
    week = now.isocalendar()[1]
    versions = [str(v.version) for v in Version.objects.filter_by_enabled(app__id=app_id)]
    data = [(v, len(WeekEvents(event_name.format(app_id, v), now.year, week))) for v in versions]
    return filter(lambda x: x[1], data)


def get_versions_data_by_platform(app_id, versions, platform, tz='UTC'):
    tzinfo = pytz.timezone(tz)
    now = timezone.now()
    start = now - timezone.timedelta(days=1)
    event_name = "online:{}:{}:{}"

    hours = [datetime(start.year, start.month, start.day, start.hour, tzinfo=pytz.UTC)
             + timezone.timedelta(hours=x) for x in range(1, 25)]

    data = [(v, [[hour.astimezone(tzinfo).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
                  len(HourEvents.from_date(event_name.format(app_id, platform, v), hour))]
                 for hour in hours])
            for v in versions]
    data = filter(lambda version_data: sum([data[1] for data in version_data[1]]), data)
    return dict(data)


def get_users_live_versions(app_id, tz='UTC'):
    win_versions = [str(v.version) for v in Version.objects.filter_by_enabled(app__id=app_id)]
    mac_versions = [str(v.short_version) for v in SparkleVersion.objects.filter_by_enabled(app__id=app_id)]

    win_data = get_versions_data_by_platform(app_id, win_versions, 'win', tz=tz)
    mac_data = get_versions_data_by_platform(app_id, mac_versions, 'mac', tz=tz)

    data = dict(win=win_data, mac=mac_data)

    return data


@valuedispatch
def is_user_active(period, userid):
    return False


@is_user_active.register(ACTIVE_USERS_DICT_CHOICES['all'])
def _(period, userid):
    return True


@is_user_active.register(ACTIVE_USERS_DICT_CHOICES['week'])
def _(period, userid):
    return get_id(userid) in WeekEvents.from_date('request', timezone.now())


@is_user_active.register(ACTIVE_USERS_DICT_CHOICES['month'])
def _(period, userid):
    return get_id(userid) in MonthEvents.from_date('request', timezone.now())


def get_kwargs_for_model(cls, obj, exclude=None):
    exclude = exclude or []
    fields = [(field.name, field.to_python) for field in cls._meta.fields if field.name not in exclude]
    kwargs = dict([(i, convert(obj.get(i))) for (i, convert) in fields])
    return kwargs


def parse_os(os):
    kwargs = get_kwargs_for_model(Os, os, exclude=['id'])
    obj, flag = Os.objects.get_or_create(**kwargs)
    return obj


def parse_hw(hw):
    kwargs = get_kwargs_for_model(Hw, hw, exclude=['id'])
    obj, flag = Hw.objects.get_or_create(**kwargs)
    return obj


def parse_req(request, ip=None):
    kwargs = get_kwargs_for_model(Request, request, exclude=['os', 'hw', 'created', 'id'])
    kwargs['ip'] = ip
    return Request(**kwargs)


def parse_apps(apps, request):
    app_list = []
    for app in apps:
        events = app.findall('event')

        if not events:
            continue

        kwargs = get_kwargs_for_model(AppRequest, app, exclude=['request', 'version', 'nextversion', 'id'])
        kwargs['version'] = app.get('version') or None
        kwargs['nextversion'] = app.get('nextversion') or None
        app_req = AppRequest.objects.create(request=request, **kwargs)
        event_list = parse_events(events)
        app_req.events.add(*event_list)
        app_list.append(app_req)
    return app_list


def parse_events(events):
    res = []
    for event in events:
        kwargs = get_kwargs_for_model(Event, event)
        res.append(Event.objects.create(**kwargs))
    return res


@transaction.atomic
def collect_statistics(request, ip=None):
    userid = request.get('userid')
    apps = request.findall('app')

    if userid:
        userid_counting(userid, apps, request.os.get('platform'))
        update_live_statistics(userid, apps, request.os.get('platform'))

    if not filter(lambda app: bool(app.findall('event')), apps):
        return

    req = parse_req(request, ip)
    req.os = parse_os(request.os)
    req.hw = parse_hw(request.hw) if request.get('hw') else None
    req.save()

    parse_apps(apps, req)



1			# coding: utf8
2
3			"""
4			This software is licensed under the Apache 2 license, quoted below.
5
6			Copyright 2014 Crystalnix Limited
7
8			Licensed under the Apache License, Version 2.0 (the "License"); you may not
9			use this file except in compliance with the License. You may obtain a copy of
10			the License at
11
12			http://www.apache.org/licenses/LICENSE-2.0
13
14			Unless required by applicable law or agreed to in writing, software
15			distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16			WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
17			License for the specific language governing permissions and limitations under
18			the License.
19			"""
20
21			from builtins import range
22
23			from functools import partial
24			from datetime import datetime, timedelta
25
26			from django.conf import settings
27			from django.db import transaction
28			from django.utils import timezone
29			from bitmapist import setup_redis, mark_event, unmark_event, WeekEvents, MonthEvents, DayEvents, HourEvents
30			import pytz
31
32			from omaha.utils import get_id, valuedispatch
33			from omaha.settings import DEFAULT_CHANNEL
34			from omaha.models import ACTIVE_USERS_DICT_CHOICES, Request, AppRequest, Os, Hw, Event, Version, Channel
35			from sparkle.models import SparkleVersion
36
37			__all__ = ['userid_counting', 'is_user_active']
38
39			host, port, db = settings.CACHES['statistics']['LOCATION'].split(':')
40			setup_redis('default', host, port, db=db)
41
42
43			def userid_counting(userid, apps_list, platform, now=None):
44			id = get_id(userid)
45			mark_event('request', id, now=now)
46			list(map(partial(add_app_statistics, id, platform, now=now), apps_list or []))
47
48
49			def add_app_statistics(userid, platform, app, now=None):
50			mark = partial(mark_event, now=now)
51			appid = app.get('appid')
52			version = app.get('version')
53			channel = app.get('tag') or DEFAULT_CHANNEL
54			mark('request:%s' % appid, userid)
55			mark('request:{}:{}'.format(appid, version), userid)
56			mark('request:{}:{}'.format(appid, platform), userid)
57			mark('request:{}:{}'.format(appid, channel), userid)
58			mark('request:{}:{}:{}'.format(appid, platform, version), userid)
59
60			def update_live_statistics(userid, apps_list, platform, now=None):
61			id = get_id(userid)
62			list(map(partial(add_app_live_statistics, id, platform, now=now), apps_list or []))
63
64
65			def add_app_live_statistics(userid, platform, app, now=None):
66			mark = partial(mark_event, now=now, track_hourly=True)
67			unmark = partial(unmark_event, track_hourly=True)
68			appid = app.get('appid')
69			version = app.get('version')
70			events = app.findall('event')
71			nextversion = app.get('nextversion')
72
73			install_event = filter(lambda x: x.get('eventtype') == '2', events)
74			if install_event and install_event[0].get('eventresult') == '1':
75			mark('online:{}:{}'.format(appid, nextversion), userid)
76			mark('online:{}:{}:{}'.format(appid, platform, nextversion), userid)
77			return
78
79			update_event = filter(lambda x: x.get('eventtype') == '3', events)
80			if update_event and update_event[0].get('eventresult') == '1':
81			unmark('online:{}:{}'.format(appid, version), userid) # necessary for
82			unmark('online:{}:{}:{}'.format(appid, platform, version), userid) # 1 hour interval
83			mark('online:{}:{}'.format(appid, nextversion), userid)
84			mark('online:{}:{}:{}'.format(appid, platform, nextversion), userid)
85			return
86
87			uninstall_event = filter(lambda x: x.get('eventtype') == '4', events)
88			if uninstall_event and uninstall_event[0].get('eventresult') == '1':
89			unmark('online:{}:{}'.format(appid, version), userid) # necessary for
90			unmark('online:{}:{}:{}'.format(appid, platform, version), userid) # 1 hour interval
91			return
92
93			# updatecheck handling
94			if version:
95			mark('online:{}:{}'.format(appid, version), userid)
96			mark('online:{}:{}:{}'.format(appid, platform, version), userid)
97
98			def get_users_statistics_months(app_id=None, year=None, start=1, end=12):
99			now = timezone.now()
100			if not year:
101			year = now.year
102			event_name = 'request:%s' % app_id if app_id else 'request'
103
104			months = []
105			for m in range(start, end + 1):
106			months.append(MonthEvents(event_name, year, m))
107			data = [(datetime(year, start + i, 1).strftime("%Y-%m"), len(e)) for i, e in enumerate(months)]
108			return data
109
110
111			def get_users_statistics_weeks(app_id=None):
112			now = timezone.now()
113			event_name = 'request:%s' % app_id if app_id else 'request'
114			year = now.year
115			current_week = now.isocalendar()[1]
116			previous_week = (now - timedelta(weeks=1)).isocalendar()[1]
117			yesterday = now - timedelta(days=1)
118			data = [
119			('Previous week', len(WeekEvents(event_name, year, previous_week))),
120			('Current week', len(WeekEvents(event_name, year, current_week))),
121			('Yesterday', len(DayEvents(event_name, year, yesterday.month, yesterday.day))),
122			('Today', len(DayEvents(event_name, year, now.month, now.day))),
123			]
124			return data
125
126
127			def get_channel_statistics(app_id):
128			now = timezone.now()
129			event_name = 'request:{}:{}'
130			week = now.isocalendar()[1]
131			channels = [c.name for c in Channel.objects.all()]
132			data = [(channel, len(WeekEvents(event_name.format(app_id, channel), now.year, week))) for channel in channels]
133			return data
134
135
136			def get_users_versions(app_id):
137			now = timezone.now()
138			event_name = 'request:{}:{}'
139			week = now.isocalendar()[1]
140			versions = [str(v.version) for v in Version.objects.filter_by_enabled(app__id=app_id)]
141			data = [(v, len(WeekEvents(event_name.format(app_id, v), now.year, week))) for v in versions]
142			return filter(lambda x: x[1], data)
143
144
145			def get_versions_data_by_platform(app_id, versions, platform, tz='UTC'):
146			tzinfo = pytz.timezone(tz)
147			now = timezone.now()
148			start = now - timezone.timedelta(days=1)
149			event_name = "online:{}:{}:{}"
150
151			hours = [datetime(start.year, start.month, start.day, start.hour, tzinfo=pytz.UTC)
152			+ timezone.timedelta(hours=x) for x in range(1, 25)]
153
154			data = [(v, [[hour.astimezone(tzinfo).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
155			len(HourEvents.from_date(event_name.format(app_id, platform, v), hour))]
156			for hour in hours])
157			for v in versions]
158			data = filter(lambda version_data: sum([data[1] for data in version_data[1]]), data)
159			return dict(data)
160
161
162			def get_users_live_versions(app_id, tz='UTC'):
163			win_versions = [str(v.version) for v in Version.objects.filter_by_enabled(app__id=app_id)]
164			mac_versions = [str(v.short_version) for v in SparkleVersion.objects.filter_by_enabled(app__id=app_id)]
165
166			win_data = get_versions_data_by_platform(app_id, win_versions, 'win', tz=tz)
167			mac_data = get_versions_data_by_platform(app_id, mac_versions, 'mac', tz=tz)
168
169			data = dict(win=win_data, mac=mac_data)
170
171			return data
172
173
174			@valuedispatch
175			def is_user_active(period, userid):
176			return False
177
178
179			@is_user_active.register(ACTIVE_USERS_DICT_CHOICES['all'])
180			def _(period, userid):
181			return True
182
183
184			@is_user_active.register(ACTIVE_USERS_DICT_CHOICES['week'])
185			def _(period, userid):
186			return get_id(userid) in WeekEvents.from_date('request', timezone.now())
187
188
189			@is_user_active.register(ACTIVE_USERS_DICT_CHOICES['month'])
190			def _(period, userid):
191			return get_id(userid) in MonthEvents.from_date('request', timezone.now())
192
193
194			def get_kwargs_for_model(cls, obj, exclude=None):
195			exclude = exclude or []
196			fields = [(field.name, field.to_python) for field in cls._meta.fields if field.name not in exclude]
197			kwargs = dict([(i, convert(obj.get(i))) for (i, convert) in fields])
198			return kwargs
199
200
201			def parse_os(os):
202			kwargs = get_kwargs_for_model(Os, os, exclude=['id'])
203			obj, flag = Os.objects.get_or_create(**kwargs)
204			return obj
205
206
207			def parse_hw(hw):
208			kwargs = get_kwargs_for_model(Hw, hw, exclude=['id'])
209			obj, flag = Hw.objects.get_or_create(**kwargs)
210			return obj
211
212
213			def parse_req(request, ip=None):
214			kwargs = get_kwargs_for_model(Request, request, exclude=['os', 'hw', 'created', 'id'])
215			kwargs['ip'] = ip
216			return Request(**kwargs)
217
218
219			def parse_apps(apps, request):
220			app_list = []
221			for app in apps:
222			events = app.findall('event')
223
224			if not events:
225			continue
226
227			kwargs = get_kwargs_for_model(AppRequest, app, exclude=['request', 'version', 'nextversion', 'id'])
228			kwargs['version'] = app.get('version') or None
229			kwargs['nextversion'] = app.get('nextversion') or None
230			app_req = AppRequest.objects.create(request=request, **kwargs)
231			event_list = parse_events(events)
232			app_req.events.add(*event_list)
233			app_list.append(app_req)
234			return app_list
235
236
237			def parse_events(events):
238			res = []
239			for event in events:
240			kwargs = get_kwargs_for_model(Event, event)
241			res.append(Event.objects.create(**kwargs))
242			return res
243
244
245			@transaction.atomic
246			def collect_statistics(request, ip=None):
247			userid = request.get('userid')
248			apps = request.findall('app')
249
250			if userid:
251			userid_counting(userid, apps, request.os.get('platform'))
252			update_live_statistics(userid, apps, request.os.get('platform'))
253
254			if not filter(lambda app: bool(app.findall('event')), apps):
255			return
256
257			req = parse_req(request, ip)
258			req.os = parse_os(request.os)
259			req.hw = parse_hw(request.hw) if request.get('hw') else None
260			req.save()
261
262			parse_apps(apps, req)
263
264

Crystalnix / omaha-server

Push — master ( b7eb8c...589bde )

collect_statistics() B

Complexity

Size

Duplication

Duplication Side-by-Side

Filter issues like