_get_key_info() - Code Metrics - ionelmc/django-redisboard - Measure and Improve Code Quality continuously with Scrutinizer

_get_key_info() B
last analyzed 2018-02-04 13:56 UTC

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	11
Bugs	0	Features	0

Metric	Value
cc	3
c	11
b	0
f	0
dl	0
loc	44
rs	8.8571

from logging import getLogger

from django.conf import settings
from django.core.paginator import Paginator
from django.http import HttpResponseNotFound
from django.shortcuts import render
from django.utils.functional import curry
from redis.exceptions import ResponseError

from .utils import PY3
from .utils import LazySlicingIterable

try:
    from django.utils.datastructures import SortedDict as OrderedDict
except ImportError:
    from django.utils.datastructures import OrderedDict
    


logger = getLogger(__name__)

REDISBOARD_ITEMS_PER_PAGE = getattr(settings, 'REDISBOARD_ITEMS_PER_PAGE', 100)


def safeint(value):
    try:
        return int(value)
    except ValueError:
        return value


def _fixup_pair(pair):
    a, b = pair
    return a, safeint(b)


LENGTH_GETTERS = {
    b'list': lambda conn, key: conn.llen(key),
    b'string': lambda conn, key: conn.strlen(key),
    b'set': lambda conn, key: conn.scard(key),
    b'zset': lambda conn, key: conn.zcount(key, '-inf', '+inf'),
    b'hash': lambda conn, key: conn.hlen(key),
}


def _get_key_info(conn, key):
    try:
        obj_type = conn.type(key)
        pipe = conn.pipeline()
        try:
            pipe.object('REFCOUNT', key)
            pipe.object('ENCODING', key)
            pipe.object('IDLETIME', key)
            LENGTH_GETTERS[obj_type](pipe, key)
            pipe.ttl(key)

            refcount, encoding, idletime, obj_length, obj_ttl = pipe.execute()
        except ResponseError as exc:
            logger.exception("Failed to get object info for key %r: %s", key, exc)
            return {
                'type': obj_type,
                'name': key,
                'length': "n/a",
                'error': str(exc),
                'ttl': "n/a",
                'refcount': "n/a",
                'encoding': "n/a",
                'idletime': "n/a",
            }
        return {
            'type': obj_type,
            'name': key,
            'length': obj_length,
            'ttl': obj_ttl,
            'refcount': refcount,
            'encoding': encoding,
            'idletime': idletime,
        }
    except ResponseError as exc:
        logger.exception("Failed to get details for key %r: %s", key, exc)
        return {
            'type': "n/a",
            'length': "n/a",
            'name': key,
            'error': str(exc),
            'ttl': "n/a",
            'refcount': "n/a",
            'encoding': "n/a",
            'idletime': "n/a",
        }

VALUE_GETTERS = {
    b'list': lambda conn, key, start=0, end=-1: [(pos + start, val)
                                                 for (pos, val) in enumerate(conn.lrange(key, start, end))],
    b'string': lambda conn, key, *args: [('string', conn.get(key))],
    b'set': lambda conn, key, *args: list(enumerate(conn.smembers(key))),
    b'zset': lambda conn, key, start=0, end=-1: [(pos + start, val)
                                                 for (pos, val) in enumerate(conn.zrange(key, start, end))],
    b'hash': lambda conn, key, *args: conn.hgetall(key).items(),
    b'n/a': lambda conn, key, *args: (),
}


def _get_key_details(conn, db, key, page):
    conn.execute_command('SELECT', db)
    details = _get_key_info(conn, key)
    details['db'] = db
    if details['type'] in ('list', 'zset'):
        details['data'] = Paginator(
            LazySlicingIterable(
                lambda: details['length'],
                curry(VALUE_GETTERS[details['type']], conn, key)
            ),
            REDISBOARD_ITEMS_PER_PAGE
        ).page(page)
    else:
        details['data'] = VALUE_GETTERS[details['type']](conn, key)

    return details

def _raw_get_db_summary(server, db):
    server.connection.execute_command('SELECT', db)
    pipe = server.connection.pipeline()

    pipe.dbsize()
    for i in range(server.sampling_threshold):
        pipe.randomkey()

    results = pipe.execute()
    size = results.pop(0)
    keys = sorted(set(results))

    pipe = server.connection.pipeline()
    for key in keys:
        pipe.execute_command('DEBUG', 'OBJECT', key)
        pipe.ttl(key)

    total_memory = 0
    volatile_memory = 0
    persistent_memory = 0
    total_keys = 0
    volatile_keys = 0
    persistent_keys = 0
    results = pipe.execute()
    for key, details, ttl in zip(keys, results[::2], results[1::2]):
        if not isinstance(details, dict):
            details = dict(_fixup_pair(i.split(b':'))
                           for i in details.split() if b':' in i)

        length = details[b'serializedlength'] + len(key)

        if ttl:
            persistent_memory += length
            persistent_keys += 1
        else:
            volatile_memory += length
            volatile_keys += 1
        total_memory += length
        total_keys += 1

    if total_keys:
        total_memory = (total_memory / total_keys) * size
    else:
        total_memory = 0

    if persistent_keys:
        persistent_memory = (persistent_memory / persistent_keys) * size
    else:
        persistent_memory = 0

    if volatile_keys:
        volatile_memory = (volatile_memory / volatile_keys) * size
    else:
        volatile_memory = 0
    return dict(
        size=size,
        total_memory=total_memory,
        volatile_memory=volatile_memory,
        persistent_memory=persistent_memory,
    )

def _get_db_summary(server, db):
    try:
        return _raw_get_db_summary(server, db)
    except ResponseError as exc:
        logger.exception("Failed to get summary for db %r: %s", db, exc)
        return dict(
            size=0,
            total_memory=0,
            volatile_memory=0,
            persistent_memory=0,
        )


def _get_db_details(server, db):
    conn = server.connection
    conn.execute_command('SELECT', db)
    size = conn.dbsize()

    key_details = {}
    if size > server.sampling_threshold:
        sampling = True
        pipe = conn.pipeline()
        for _ in (range if PY3 else xrange)(server.sampling_size):  # flake8=noqa
            pipe.randomkey()

        for key in set(pipe.execute()):
            key_details[key] = _get_key_info(conn, key)

    else:
        sampling = False
        for key in conn.keys():
            key_details[key] = _get_key_info(conn, key)

    return dict(
        keys=key_details,
        sampling=sampling,
    )


def inspect(request, server):
    stats = server.stats
    conn = server.connection
    database_details = OrderedDict()
    key_details = None

    if stats['status'] == 'UP':
        if 'key' in request.GET:
            key = request.GET['key']
            db = request.GET.get('db', 0)
            page = request.GET.get('page', 1)
            key_details = _get_key_details(conn, db, key, page)
        else:
            databases = sorted(name[2:] for name in conn.info()
                               if name.startswith('db'))
            total_size = 0
            for db in databases:
                database_details[db] = summary = _get_db_summary(server, db)
                total_size += summary['size']
            if total_size < server.sampling_threshold:
                for db in databases:
                    database_details[db].update(
                        _get_db_details(server, db),
                        active=True,
                    )
            elif 'db' in request.GET:
                db = request.GET['db']
                if db in database_details:
                    database_details[db].update(
                        _get_db_details(server, db),
                        active=True,
                    )
                else:
                    return HttpResponseNotFound("Unknown database.")

    return render(request, "redisboard/inspect.html", {
        'databases': database_details,
        'key_details': key_details,
        'original': server,
        'stats': stats,
        'app_label': 'redisboard',
    })


1			from logging import getLogger
2
3			from django.conf import settings
4			from django.core.paginator import Paginator
5			from django.http import HttpResponseNotFound
6			from django.shortcuts import render
7			from django.utils.functional import curry
8			from redis.exceptions import ResponseError
9
10			from .utils import PY3
11			from .utils import LazySlicingIterable
12
13			try:
14			from django.utils.datastructures import SortedDict as OrderedDict
15			except ImportError:
16			from django.utils.datastructures import OrderedDict
17
18
19
20			logger = getLogger(__name__)
21
22			REDISBOARD_ITEMS_PER_PAGE = getattr(settings, 'REDISBOARD_ITEMS_PER_PAGE', 100)
23
24
25			def safeint(value):
26			try:
27			return int(value)
28			except ValueError:
29			return value
30
31
32			def _fixup_pair(pair):
33			a, b = pair
34			return a, safeint(b)
35
36
37			LENGTH_GETTERS = {
38			b'list': lambda conn, key: conn.llen(key),
39			b'string': lambda conn, key: conn.strlen(key),
40			b'set': lambda conn, key: conn.scard(key),
41			b'zset': lambda conn, key: conn.zcount(key, '-inf', '+inf'),
42			b'hash': lambda conn, key: conn.hlen(key),
43			}
44
45
46			def _get_key_info(conn, key):
47			try:
48			obj_type = conn.type(key)
49			pipe = conn.pipeline()
50			try:
51			pipe.object('REFCOUNT', key)
52			pipe.object('ENCODING', key)
53			pipe.object('IDLETIME', key)
54			LENGTH_GETTERS[obj_type](pipe, key)
55			pipe.ttl(key)
56
57			refcount, encoding, idletime, obj_length, obj_ttl = pipe.execute()
58			except ResponseError as exc:
59			logger.exception("Failed to get object info for key %r: %s", key, exc)
60			return {
61			'type': obj_type,
62			'name': key,
63			'length': "n/a",
64			'error': str(exc),
65			'ttl': "n/a",
66			'refcount': "n/a",
67			'encoding': "n/a",
68			'idletime': "n/a",
69			}
70			return {
71			'type': obj_type,
72			'name': key,
73			'length': obj_length,
74			'ttl': obj_ttl,
75			'refcount': refcount,
76			'encoding': encoding,
77			'idletime': idletime,
78			}
79			except ResponseError as exc:
80			logger.exception("Failed to get details for key %r: %s", key, exc)
81			return {
82			'type': "n/a",
83			'length': "n/a",
84			'name': key,
85			'error': str(exc),
86			'ttl': "n/a",
87			'refcount': "n/a",
88			'encoding': "n/a",
89			'idletime': "n/a",
90			}
91
92			VALUE_GETTERS = {
93			b'list': lambda conn, key, start=0, end=-1: [(pos + start, val)
94			for (pos, val) in enumerate(conn.lrange(key, start, end))],
95			b'string': lambda conn, key, *args: [('string', conn.get(key))],
96			b'set': lambda conn, key, *args: list(enumerate(conn.smembers(key))),
97			b'zset': lambda conn, key, start=0, end=-1: [(pos + start, val)
98			for (pos, val) in enumerate(conn.zrange(key, start, end))],
99			b'hash': lambda conn, key, *args: conn.hgetall(key).items(),
100			b'n/a': lambda conn, key, *args: (),
101			}
102
103
104			def _get_key_details(conn, db, key, page):
105			conn.execute_command('SELECT', db)
106			details = _get_key_info(conn, key)
107			details['db'] = db
108			if details['type'] in ('list', 'zset'):
109			details['data'] = Paginator(
110			LazySlicingIterable(
111			lambda: details['length'],
112			curry(VALUE_GETTERS[details['type']], conn, key)
113			),
114			REDISBOARD_ITEMS_PER_PAGE
115			).page(page)
116			else:
117			details['data'] = VALUE_GETTERS[details['type']](conn, key)
118
119			return details
120
121			def _raw_get_db_summary(server, db):
122			server.connection.execute_command('SELECT', db)
123			pipe = server.connection.pipeline()
124
125			pipe.dbsize()
126			for i in range(server.sampling_threshold):
127			pipe.randomkey()
128
129			results = pipe.execute()
130			size = results.pop(0)
131			keys = sorted(set(results))
132
133			pipe = server.connection.pipeline()
134			for key in keys:
135			pipe.execute_command('DEBUG', 'OBJECT', key)
136			pipe.ttl(key)
137
138			total_memory = 0
139			volatile_memory = 0
140			persistent_memory = 0
141			total_keys = 0
142			volatile_keys = 0
143			persistent_keys = 0
144			results = pipe.execute()
145			for key, details, ttl in zip(keys, results[::2], results[1::2]):
146			if not isinstance(details, dict):
147			details = dict(_fixup_pair(i.split(b':'))
148			for i in details.split() if b':' in i)
149
150			length = details[b'serializedlength'] + len(key)
151
152			if ttl:
153			persistent_memory += length
154			persistent_keys += 1
155			else:
156			volatile_memory += length
157			volatile_keys += 1
158			total_memory += length
159			total_keys += 1
160
161			if total_keys:
162			total_memory = (total_memory / total_keys) * size
163			else:
164			total_memory = 0
165
166			if persistent_keys:
167			persistent_memory = (persistent_memory / persistent_keys) * size
168			else:
169			persistent_memory = 0
170
171			if volatile_keys:
172			volatile_memory = (volatile_memory / volatile_keys) * size
173			else:
174			volatile_memory = 0
175			return dict(
176			size=size,
177			total_memory=total_memory,
178			volatile_memory=volatile_memory,
179			persistent_memory=persistent_memory,
180			)
181
182			def _get_db_summary(server, db):
183			try:
184			return _raw_get_db_summary(server, db)
185			except ResponseError as exc:
186			logger.exception("Failed to get summary for db %r: %s", db, exc)
187			return dict(
188			size=0,
189			total_memory=0,
190			volatile_memory=0,
191			persistent_memory=0,
192			)
193
194
195			def _get_db_details(server, db):
196			conn = server.connection
197			conn.execute_command('SELECT', db)
198			size = conn.dbsize()
199
200			key_details = {}
201			if size > server.sampling_threshold:
202			sampling = True
203			pipe = conn.pipeline()
204			for _ in (range if PY3 else xrange)(server.sampling_size): # flake8=noqa
205			pipe.randomkey()
206
207			for key in set(pipe.execute()):
208			key_details[key] = _get_key_info(conn, key)
209
210			else:
211			sampling = False
212			for key in conn.keys():
213			key_details[key] = _get_key_info(conn, key)
214
215			return dict(
216			keys=key_details,
217			sampling=sampling,
218			)
219
220
221			def inspect(request, server):
222			stats = server.stats
223			conn = server.connection
224			database_details = OrderedDict()
225			key_details = None
226
227			if stats['status'] == 'UP':
228			if 'key' in request.GET:
229			key = request.GET['key']
230			db = request.GET.get('db', 0)
231			page = request.GET.get('page', 1)
232			key_details = _get_key_details(conn, db, key, page)
233			else:
234			databases = sorted(name[2:] for name in conn.info()
235			if name.startswith('db'))
236			total_size = 0
237			for db in databases:
238			database_details[db] = summary = _get_db_summary(server, db)
239			total_size += summary['size']
240			if total_size < server.sampling_threshold:
241			for db in databases:
242			database_details[db].update(
243			_get_db_details(server, db),
244			active=True,
245			)
246			elif 'db' in request.GET:
247			db = request.GET['db']
248			if db in database_details:
249			database_details[db].update(
250			_get_db_details(server, db),
251			active=True,
252			)
253			else:
254			return HttpResponseNotFound("Unknown database.")
255
256			return render(request, "redisboard/inspect.html", {
257			'databases': database_details,
258			'key_details': key_details,
259			'original': server,
260			'stats': stats,
261			'app_label': 'redisboard',
262			})
263

ionelmc / django-redisboard

_get_key_info() B last analyzed 2018-02-04 13:56 UTC

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like

_get_key_info() B
last analyzed 2018-02-04 13:56 UTC