_find_subqueries() - Code Metrics - Inspection of "Tests schema changing queries." - noripyt/django-cachalot - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( b5827c...809ef6 )

by Bertrand

created 2017-06-03 20:01 UTC

_find_subqueries() C

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	7
Bugs	0	Features	0

Metric	Value
cc	10
c	7
b	0
f	0
dl	0
loc	21
rs	5.2413

How to fix Complexity

# coding: utf-8

from __future__ import unicode_literals
import datetime
from decimal import Decimal
from hashlib import sha1
from time import time
from uuid import UUID

from django import VERSION as django_version
from django.db import connections
from django.db.models import QuerySet
from django.db.models.sql import Query
from django.db.models.sql.where import (
    ExtraWhere, SubqueryConstraint, WhereNode)
from django.utils.module_loading import import_string
from django.utils.six import text_type, binary_type, PY2

from .settings import cachalot_settings
from .transaction import AtomicCache


class UncachableQuery(Exception):
    pass


class IsRawQuery(Exception):
    pass


TUPLE_OR_LIST = {tuple, list}

CACHABLE_PARAM_TYPES = {
    bool, int, float, Decimal, bytearray, binary_type, text_type, type(None),
    datetime.date, datetime.time, datetime.datetime, datetime.timedelta, UUID,
}

if PY2:
    CACHABLE_PARAM_TYPES.add(long)

UNCACHABLE_FUNCS = set()
if django_version[:2] >= (1, 9):
    from django.db.models.functions import Now
    from django.contrib.postgres.functions import TransactionNow
    UNCACHABLE_FUNCS.update((Now, TransactionNow))

try:
    from psycopg2 import Binary
    from psycopg2.extras import (
        NumericRange, DateRange, DateTimeRange, DateTimeTZRange, Inet, Json)
except ImportError:
    pass
else:
    CACHABLE_PARAM_TYPES.update((
        Binary,
        NumericRange, DateRange, DateTimeRange, DateTimeTZRange, Inet, Json))
    if django_version[:2] >= (1, 11):
        from django.contrib.postgres.fields.jsonb import JsonAdapter
        CACHABLE_PARAM_TYPES.add(JsonAdapter)


def check_parameter_types(params):
    for p in params:
        cl = p.__class__
        if cl not in CACHABLE_PARAM_TYPES:
            if cl in TUPLE_OR_LIST:
                check_parameter_types(p)
            elif cl is dict:
                check_parameter_types(p.items())
            else:
                raise UncachableQuery


def get_query_cache_key(compiler):
    """
    Generates a cache key from a SQLCompiler.

    This cache key is specific to the SQL query and its context
    (which database is used).  The same query in the same context
    (= the same database) must generate the same cache key.

    :arg compiler: A SQLCompiler that will generate the SQL query
    :type compiler: django.db.models.sql.compiler.SQLCompiler
    :return: A cache key
    :rtype: int
    """
    sql, params = compiler.as_sql()
    check_parameter_types(params)
    cache_key = '%s:%s:%s' % (compiler.using, sql,
                              [text_type(p) for p in params])
    return sha1(cache_key.encode('utf-8')).hexdigest()


def get_table_cache_key(db_alias, table):
    """
    Generates a cache key from a SQL table.

    :arg db_alias: Alias of the used database
    :type db_alias: str or unicode
    :arg table: Name of the SQL table
    :type table: str or unicode
    :return: A cache key
    :rtype: int
    """
    cache_key = '%s:%s' % (db_alias, table)
    return sha1(cache_key.encode('utf-8')).hexdigest()


def _get_query_cache_key(compiler):
    return import_string(cachalot_settings.CACHALOT_QUERY_KEYGEN)(compiler)


def _get_table_cache_key(db_alias, table):
    return import_string(cachalot_settings.CACHALOT_TABLE_KEYGEN)(db_alias, table)


def _get_tables_from_sql(connection, lowercased_sql):
    return {t for t in connection.introspection.django_table_names()
            if t in lowercased_sql}


def _find_subqueries(children):
    for child in children:
        child_class = child.__class__
        if child_class is WhereNode:
            for grand_child in _find_subqueries(child.children):
                yield grand_child
        elif child_class is SubqueryConstraint:
            query_object = child.query_object
            yield (query_object if query_object.__class__ is Query
                   else query_object.query)
        elif child_class is ExtraWhere:
            raise IsRawQuery
        else:
            rhs = getattr(child, 'rhs', None)
            rhs_class = rhs.__class__
            if rhs_class is Query:
                yield rhs
            elif rhs_class is QuerySet:
                yield rhs.query
            elif rhs_class in UNCACHABLE_FUNCS:
                raise UncachableQuery


def is_cachable(table):
    whitelist = cachalot_settings.CACHALOT_ONLY_CACHABLE_TABLES
    if whitelist and table not in whitelist:
        return False
    return table not in cachalot_settings.CACHALOT_UNCACHABLE_TABLES


def are_all_cachable(tables):
    whitelist = cachalot_settings.CACHALOT_ONLY_CACHABLE_TABLES
    if whitelist and not tables.issubset(whitelist):
        return False
    return tables.isdisjoint(cachalot_settings.CACHALOT_UNCACHABLE_TABLES)


def filter_cachable(tables):
    whitelist = cachalot_settings.CACHALOT_ONLY_CACHABLE_TABLES
    tables = tables.difference(cachalot_settings.CACHALOT_UNCACHABLE_TABLES)
    if whitelist:
        return tables.intersection(whitelist)
    return tables


def _get_tables(db_alias, query):
    if query.select_for_update or (
            '?' in query.order_by
            and not cachalot_settings.CACHALOT_CACHE_RANDOM):
        raise UncachableQuery

    try:
        if query.extra_select or getattr(query, 'subquery', False):
            raise IsRawQuery
        tables = set(query.table_map)
        tables.add(query.get_meta().db_table)
        for subquery in _find_subqueries(query.where.children):
            tables.update(_get_tables(db_alias, subquery))
    except IsRawQuery:
        sql = query.get_compiler(db_alias).as_sql()[0].lower()
        tables = _get_tables_from_sql(connections[db_alias], sql)

    if not are_all_cachable(tables):
        raise UncachableQuery
    return tables


def _get_table_cache_keys(compiler):
    db_alias = compiler.using
    return [_get_table_cache_key(db_alias, t)
            for t in _get_tables(db_alias, compiler.query)]


def _invalidate_tables(cache, db_alias, tables):
    tables = filter_cachable(set(tables))
    if not tables:
        return
    now = time()
    cache.set_many(
        {_get_table_cache_key(db_alias, t): now for t in tables},
        cachalot_settings.CACHALOT_TIMEOUT)

    if isinstance(cache, AtomicCache):
        cache.to_be_invalidated.update(tables)


1			# coding: utf-8
2
3			from __future__ import unicode_literals
4			import datetime
5			from decimal import Decimal
6			from hashlib import sha1
7			from time import time
8			from uuid import UUID
9
10			from django import VERSION as django_version
11			from django.db import connections
12			from django.db.models import QuerySet
13			from django.db.models.sql import Query
14			from django.db.models.sql.where import (
15			ExtraWhere, SubqueryConstraint, WhereNode)
16			from django.utils.module_loading import import_string
17			from django.utils.six import text_type, binary_type, PY2
18
19			from .settings import cachalot_settings
20			from .transaction import AtomicCache
21
22
23			class UncachableQuery(Exception):
24			pass
25
26
27			class IsRawQuery(Exception):
28			pass
29
30
31			TUPLE_OR_LIST = {tuple, list}
32
33			CACHABLE_PARAM_TYPES = {
34			bool, int, float, Decimal, bytearray, binary_type, text_type, type(None),
35			datetime.date, datetime.time, datetime.datetime, datetime.timedelta, UUID,
36			}
37
38			if PY2:
39			CACHABLE_PARAM_TYPES.add(long)
40
41			UNCACHABLE_FUNCS = set()
42			if django_version[:2] >= (1, 9):
43			from django.db.models.functions import Now
44			from django.contrib.postgres.functions import TransactionNow
45			UNCACHABLE_FUNCS.update((Now, TransactionNow))
46
47			try:
48			from psycopg2 import Binary
49			from psycopg2.extras import (
50			NumericRange, DateRange, DateTimeRange, DateTimeTZRange, Inet, Json)
51			except ImportError:
52			pass
53			else:
54			CACHABLE_PARAM_TYPES.update((
55			Binary,
56			NumericRange, DateRange, DateTimeRange, DateTimeTZRange, Inet, Json))
57			if django_version[:2] >= (1, 11):
58			from django.contrib.postgres.fields.jsonb import JsonAdapter
59			CACHABLE_PARAM_TYPES.add(JsonAdapter)
60
61
62			def check_parameter_types(params):
63			for p in params:
64			cl = p.__class__
65			if cl not in CACHABLE_PARAM_TYPES:
66			if cl in TUPLE_OR_LIST:
67			check_parameter_types(p)
68			elif cl is dict:
69			check_parameter_types(p.items())
70			else:
71			raise UncachableQuery
72
73
74			def get_query_cache_key(compiler):
75			"""
76			Generates a cache key from a SQLCompiler.
77
78			This cache key is specific to the SQL query and its context
79			(which database is used). The same query in the same context
80			(= the same database) must generate the same cache key.
81
82			:arg compiler: A SQLCompiler that will generate the SQL query
83			:type compiler: django.db.models.sql.compiler.SQLCompiler
84			:return: A cache key
85			:rtype: int
86			"""
87			sql, params = compiler.as_sql()
88			check_parameter_types(params)
89			cache_key = '%s:%s:%s' % (compiler.using, sql,
90			[text_type(p) for p in params])
91			return sha1(cache_key.encode('utf-8')).hexdigest()
92
93
94			def get_table_cache_key(db_alias, table):
95			"""
96			Generates a cache key from a SQL table.
97
98			:arg db_alias: Alias of the used database
99			:type db_alias: str or unicode
100			:arg table: Name of the SQL table
101			:type table: str or unicode
102			:return: A cache key
103			:rtype: int
104			"""
105			cache_key = '%s:%s' % (db_alias, table)
106			return sha1(cache_key.encode('utf-8')).hexdigest()
107
108
109			def _get_query_cache_key(compiler):
110			return import_string(cachalot_settings.CACHALOT_QUERY_KEYGEN)(compiler)
111
112
113			def _get_table_cache_key(db_alias, table):
114			return import_string(cachalot_settings.CACHALOT_TABLE_KEYGEN)(db_alias, table)
115
116
117			def _get_tables_from_sql(connection, lowercased_sql):
118			return {t for t in connection.introspection.django_table_names()
119			if t in lowercased_sql}
120
121
122			def _find_subqueries(children):
123			for child in children:
124			child_class = child.__class__
125			if child_class is WhereNode:
126			for grand_child in _find_subqueries(child.children):
127			yield grand_child
128			elif child_class is SubqueryConstraint:
129			query_object = child.query_object
130			yield (query_object if query_object.__class__ is Query
131			else query_object.query)
132			elif child_class is ExtraWhere:
133			raise IsRawQuery
134			else:
135			rhs = getattr(child, 'rhs', None)
136			rhs_class = rhs.__class__
137			if rhs_class is Query:
138			yield rhs
139			elif rhs_class is QuerySet:
140			yield rhs.query
141			elif rhs_class in UNCACHABLE_FUNCS:
142			raise UncachableQuery
143
144
145			def is_cachable(table):
146			whitelist = cachalot_settings.CACHALOT_ONLY_CACHABLE_TABLES
147			if whitelist and table not in whitelist:
148			return False
149			return table not in cachalot_settings.CACHALOT_UNCACHABLE_TABLES
150
151
152			def are_all_cachable(tables):
153			whitelist = cachalot_settings.CACHALOT_ONLY_CACHABLE_TABLES
154			if whitelist and not tables.issubset(whitelist):
155			return False
156			return tables.isdisjoint(cachalot_settings.CACHALOT_UNCACHABLE_TABLES)
157
158
159			def filter_cachable(tables):
160			whitelist = cachalot_settings.CACHALOT_ONLY_CACHABLE_TABLES
161			tables = tables.difference(cachalot_settings.CACHALOT_UNCACHABLE_TABLES)
162			if whitelist:
163			return tables.intersection(whitelist)
164			return tables
165
166
167			def _get_tables(db_alias, query):
168			if query.select_for_update or (
169			'?' in query.order_by
170			and not cachalot_settings.CACHALOT_CACHE_RANDOM):
171			raise UncachableQuery
172
173			try:
174			if query.extra_select or getattr(query, 'subquery', False):
175			raise IsRawQuery
176			tables = set(query.table_map)
177			tables.add(query.get_meta().db_table)
178			for subquery in _find_subqueries(query.where.children):
179			tables.update(_get_tables(db_alias, subquery))
180			except IsRawQuery:
181			sql = query.get_compiler(db_alias).as_sql()[0].lower()
182			tables = _get_tables_from_sql(connections[db_alias], sql)
183
184			if not are_all_cachable(tables):
185			raise UncachableQuery
186			return tables
187
188
189			def _get_table_cache_keys(compiler):
190			db_alias = compiler.using
191			return [_get_table_cache_key(db_alias, t)
192			for t in _get_tables(db_alias, compiler.query)]
193
194
195			def _invalidate_tables(cache, db_alias, tables):
196			tables = filter_cachable(set(tables))
197			if not tables:
198			return
199			now = time()
200			cache.set_many(
201			{_get_table_cache_key(db_alias, t): now for t in tables},
202			cachalot_settings.CACHALOT_TIMEOUT)
203
204			if isinstance(cache, AtomicCache):
205			cache.to_be_invalidated.update(tables)
206

noripyt / django-cachalot

Push — master ( b5827c...809ef6 )

_find_subqueries() C

Complexity

Size

Duplication

Importance

How to fix Complexity

Complexity

Duplication Side-by-Side

Filter issues like