_find_subqueries() - Code Metrics - Inspection of "Multiple Python optimisations." - noripyt/django-cachalot - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( 78c434...bc49fd )

by Bertrand

created 2017-06-07 06:59 UTC

_find_subqueries() D

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	6
Bugs	0	Features	0

Metric	Value
cc	10
dl	0
loc	22
rs	4.5957
c	6
b	0
f	0

How to fix Complexity

# coding: utf-8

from __future__ import unicode_literals
import datetime
from decimal import Decimal
from hashlib import sha1
from time import time
from uuid import UUID

from django import VERSION as django_version
from django.db import connections
from django.db.models import QuerySet
from django.db.models.sql import Query
from django.db.models.sql.where import (
    ExtraWhere, SubqueryConstraint, WhereNode)
from django.utils.six import text_type, binary_type, PY2

from .settings import ITERABLES, cachalot_settings
from .transaction import AtomicCache


class UncachableQuery(Exception):
    pass


class IsRawQuery(Exception):
    pass


CACHABLE_PARAM_TYPES = {
    bool, int, float, Decimal, bytearray, binary_type, text_type, type(None),
    datetime.date, datetime.time, datetime.datetime, datetime.timedelta, UUID,
}

if PY2:
    CACHABLE_PARAM_TYPES.add(long)

UNCACHABLE_FUNCS = set()
if django_version[:2] >= (1, 9):
    from django.db.models.functions import Now
    from django.contrib.postgres.functions import TransactionNow
    UNCACHABLE_FUNCS.update((Now, TransactionNow))

try:
    from psycopg2 import Binary
    from psycopg2.extras import (
        NumericRange, DateRange, DateTimeRange, DateTimeTZRange, Inet, Json)
except ImportError:
    pass
else:
    CACHABLE_PARAM_TYPES.update((
        Binary,
        NumericRange, DateRange, DateTimeRange, DateTimeTZRange, Inet, Json))
    if django_version[:2] >= (1, 11):
        from django.contrib.postgres.fields.jsonb import JsonAdapter
        CACHABLE_PARAM_TYPES.add(JsonAdapter)


def check_parameter_types(params):
    for p in params:
        cl = p.__class__
        if cl not in CACHABLE_PARAM_TYPES:
            if cl in ITERABLES:
                check_parameter_types(p)
            elif cl is dict:
                check_parameter_types(p.items())
            else:
                raise UncachableQuery


def get_query_cache_key(compiler):
    """
    Generates a cache key from a SQLCompiler.

    This cache key is specific to the SQL query and its context
    (which database is used).  The same query in the same context
    (= the same database) must generate the same cache key.

    :arg compiler: A SQLCompiler that will generate the SQL query
    :type compiler: django.db.models.sql.compiler.SQLCompiler
    :return: A cache key
    :rtype: int
    """
    sql, params = compiler.as_sql()
    check_parameter_types(params)
    cache_key = '%s:%s:%s' % (compiler.using, sql,
                              [text_type(p) for p in params])
    return sha1(cache_key.encode('utf-8')).hexdigest()


def get_table_cache_key(db_alias, table):
    """
    Generates a cache key from a SQL table.

    :arg db_alias: Alias of the used database
    :type db_alias: str or unicode
    :arg table: Name of the SQL table
    :type table: str or unicode
    :return: A cache key
    :rtype: int
    """
    cache_key = '%s:%s' % (db_alias, table)
    return sha1(cache_key.encode('utf-8')).hexdigest()


def _get_tables_from_sql(connection, lowercased_sql):
    return {t for t in connection.introspection.django_table_names()
            if t in lowercased_sql}


def _find_subqueries(children):
    for child in children:
        child_class = child.__class__
        if child_class is WhereNode:
            for grand_child in _find_subqueries(child.children):
                yield grand_child
        # TODO: Remove this condition when we drop Django 1.8 support.
        elif child_class is SubqueryConstraint:
            query_object = child.query_object
            yield (query_object if query_object.__class__ is Query
                   else query_object.query)
        elif child_class is ExtraWhere:
            raise IsRawQuery
        else:
            rhs = getattr(child, 'rhs', None)
            rhs_class = rhs.__class__
            if rhs_class is Query:
                yield rhs
            elif rhs_class is QuerySet:
                yield rhs.query
            elif rhs_class in UNCACHABLE_FUNCS:
                raise UncachableQuery


def is_cachable(table):
    whitelist = cachalot_settings.CACHALOT_ONLY_CACHABLE_TABLES
    if whitelist and table not in whitelist:
        return False
    return table not in cachalot_settings.CACHALOT_UNCACHABLE_TABLES


def are_all_cachable(tables):
    whitelist = cachalot_settings.CACHALOT_ONLY_CACHABLE_TABLES
    if whitelist and not tables.issubset(whitelist):
        return False
    return tables.isdisjoint(cachalot_settings.CACHALOT_UNCACHABLE_TABLES)


def filter_cachable(tables):
    whitelist = cachalot_settings.CACHALOT_ONLY_CACHABLE_TABLES
    tables = tables.difference(cachalot_settings.CACHALOT_UNCACHABLE_TABLES)
    if whitelist:
        return tables.intersection(whitelist)
    return tables


def _get_tables(db_alias, query):
    if query.select_for_update or (
            not cachalot_settings.CACHALOT_CACHE_RANDOM
            and '?' in query.order_by):
        raise UncachableQuery

    try:
        if query.extra_select or getattr(query, 'subquery', False):
            raise IsRawQuery
        tables = set(query.table_map)
        tables.add(query.get_meta().db_table)
        for subquery in _find_subqueries(query.where.children):
            tables.update(_get_tables(db_alias, subquery))
    except IsRawQuery:
        sql = query.get_compiler(db_alias).as_sql()[0].lower()
        tables = _get_tables_from_sql(connections[db_alias], sql)

    if not are_all_cachable(tables):
        raise UncachableQuery
    return tables


def _get_table_cache_keys(compiler):
    db_alias = compiler.using
    get_table_cache_key = cachalot_settings.CACHALOT_TABLE_KEYGEN
    return [get_table_cache_key(db_alias, t)
            for t in _get_tables(db_alias, compiler.query)]


def _invalidate_tables(cache, db_alias, tables):
    tables = filter_cachable(set(tables))
    if not tables:
        return
    now = time()
    get_table_cache_key = cachalot_settings.CACHALOT_TABLE_KEYGEN
    cache.set_many(
        {get_table_cache_key(db_alias, t): now for t in tables},
        cachalot_settings.CACHALOT_TIMEOUT)

    if isinstance(cache, AtomicCache):
        cache.to_be_invalidated.update(tables)


1			# coding: utf-8
2
3			from __future__ import unicode_literals
4			import datetime
5			from decimal import Decimal
6			from hashlib import sha1
7			from time import time
8			from uuid import UUID
9
10			from django import VERSION as django_version
11			from django.db import connections
12			from django.db.models import QuerySet
13			from django.db.models.sql import Query
14			from django.db.models.sql.where import (
15			ExtraWhere, SubqueryConstraint, WhereNode)
16			from django.utils.six import text_type, binary_type, PY2
17
18			from .settings import ITERABLES, cachalot_settings
19			from .transaction import AtomicCache
20
21
22			class UncachableQuery(Exception):
23			pass
24
25
26			class IsRawQuery(Exception):
27			pass
28
29
30			CACHABLE_PARAM_TYPES = {
31			bool, int, float, Decimal, bytearray, binary_type, text_type, type(None),
32			datetime.date, datetime.time, datetime.datetime, datetime.timedelta, UUID,
33			}
34
35			if PY2:
36			CACHABLE_PARAM_TYPES.add(long)
37
38			UNCACHABLE_FUNCS = set()
39			if django_version[:2] >= (1, 9):
40			from django.db.models.functions import Now
41			from django.contrib.postgres.functions import TransactionNow
42			UNCACHABLE_FUNCS.update((Now, TransactionNow))
43
44			try:
45			from psycopg2 import Binary
46			from psycopg2.extras import (
47			NumericRange, DateRange, DateTimeRange, DateTimeTZRange, Inet, Json)
48			except ImportError:
49			pass
50			else:
51			CACHABLE_PARAM_TYPES.update((
52			Binary,
53			NumericRange, DateRange, DateTimeRange, DateTimeTZRange, Inet, Json))
54			if django_version[:2] >= (1, 11):
55			from django.contrib.postgres.fields.jsonb import JsonAdapter
56			CACHABLE_PARAM_TYPES.add(JsonAdapter)
57
58
59			def check_parameter_types(params):
60			for p in params:
61			cl = p.__class__
62			if cl not in CACHABLE_PARAM_TYPES:
63			if cl in ITERABLES:
64			check_parameter_types(p)
65			elif cl is dict:
66			check_parameter_types(p.items())
67			else:
68			raise UncachableQuery
69
70
71			def get_query_cache_key(compiler):
72			"""
73			Generates a cache key from a SQLCompiler.
74
75			This cache key is specific to the SQL query and its context
76			(which database is used). The same query in the same context
77			(= the same database) must generate the same cache key.
78
79			:arg compiler: A SQLCompiler that will generate the SQL query
80			:type compiler: django.db.models.sql.compiler.SQLCompiler
81			:return: A cache key
82			:rtype: int
83			"""
84			sql, params = compiler.as_sql()
85			check_parameter_types(params)
86			cache_key = '%s:%s:%s' % (compiler.using, sql,
87			[text_type(p) for p in params])
88			return sha1(cache_key.encode('utf-8')).hexdigest()
89
90
91			def get_table_cache_key(db_alias, table):
92			"""
93			Generates a cache key from a SQL table.
94
95			:arg db_alias: Alias of the used database
96			:type db_alias: str or unicode
97			:arg table: Name of the SQL table
98			:type table: str or unicode
99			:return: A cache key
100			:rtype: int
101			"""
102			cache_key = '%s:%s' % (db_alias, table)
103			return sha1(cache_key.encode('utf-8')).hexdigest()
104
105
106			def _get_tables_from_sql(connection, lowercased_sql):
107			return {t for t in connection.introspection.django_table_names()
108			if t in lowercased_sql}
109
110
111			def _find_subqueries(children):
112			for child in children:
113			child_class = child.__class__
114			if child_class is WhereNode:
115			for grand_child in _find_subqueries(child.children):
116			yield grand_child
117			# TODO: Remove this condition when we drop Django 1.8 support.
118			elif child_class is SubqueryConstraint:
119			query_object = child.query_object
120			yield (query_object if query_object.__class__ is Query
121			else query_object.query)
122			elif child_class is ExtraWhere:
123			raise IsRawQuery
124			else:
125			rhs = getattr(child, 'rhs', None)
126			rhs_class = rhs.__class__
127			if rhs_class is Query:
128			yield rhs
129			elif rhs_class is QuerySet:
130			yield rhs.query
131			elif rhs_class in UNCACHABLE_FUNCS:
132			raise UncachableQuery
133
134
135			def is_cachable(table):
136			whitelist = cachalot_settings.CACHALOT_ONLY_CACHABLE_TABLES
137			if whitelist and table not in whitelist:
138			return False
139			return table not in cachalot_settings.CACHALOT_UNCACHABLE_TABLES
140
141
142			def are_all_cachable(tables):
143			whitelist = cachalot_settings.CACHALOT_ONLY_CACHABLE_TABLES
144			if whitelist and not tables.issubset(whitelist):
145			return False
146			return tables.isdisjoint(cachalot_settings.CACHALOT_UNCACHABLE_TABLES)
147
148
149			def filter_cachable(tables):
150			whitelist = cachalot_settings.CACHALOT_ONLY_CACHABLE_TABLES
151			tables = tables.difference(cachalot_settings.CACHALOT_UNCACHABLE_TABLES)
152			if whitelist:
153			return tables.intersection(whitelist)
154			return tables
155
156
157			def _get_tables(db_alias, query):
158			if query.select_for_update or (
159			not cachalot_settings.CACHALOT_CACHE_RANDOM
160			and '?' in query.order_by):
161			raise UncachableQuery
162
163			try:
164			if query.extra_select or getattr(query, 'subquery', False):
165			raise IsRawQuery
166			tables = set(query.table_map)
167			tables.add(query.get_meta().db_table)
168			for subquery in _find_subqueries(query.where.children):
169			tables.update(_get_tables(db_alias, subquery))
170			except IsRawQuery:
171			sql = query.get_compiler(db_alias).as_sql()[0].lower()
172			tables = _get_tables_from_sql(connections[db_alias], sql)
173
174			if not are_all_cachable(tables):
175			raise UncachableQuery
176			return tables
177
178
179			def _get_table_cache_keys(compiler):
180			db_alias = compiler.using
181			get_table_cache_key = cachalot_settings.CACHALOT_TABLE_KEYGEN
182			return [get_table_cache_key(db_alias, t)
183			for t in _get_tables(db_alias, compiler.query)]
184
185
186			def _invalidate_tables(cache, db_alias, tables):
187			tables = filter_cachable(set(tables))
188			if not tables:
189			return
190			now = time()
191			get_table_cache_key = cachalot_settings.CACHALOT_TABLE_KEYGEN
192			cache.set_many(
193			{get_table_cache_key(db_alias, t): now for t in tables},
194			cachalot_settings.CACHALOT_TIMEOUT)
195
196			if isinstance(cache, AtomicCache):
197			cache.to_be_invalidated.update(tables)
198

noripyt / django-cachalot

Push — master ( 78c434...bc49fd )

_find_subqueries() D

Complexity

Size

Duplication

Importance

How to fix Complexity

Complexity

Duplication Side-by-Side

Filter issues like