_find_subqueries_in_where() - Code Metrics - Inspection of "Adds Django 2.0 support." - noripyt/django-cachalot - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( 6e429f...74e000 )

by Bertrand

created 2018-05-04 18:26 UTC

_find_subqueries_in_where() C

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	10
dl	0
loc	19
rs	6
c	0
b	0
f	0

How to fix Complexity

# coding: utf-8

from __future__ import unicode_literals
import datetime
from decimal import Decimal
from hashlib import sha1
from time import time
from uuid import UUID

from django.contrib.postgres.functions import TransactionNow
from django.db import connections
from django.db.models import QuerySet, Subquery, Exists
from django.db.models.functions import Now
from django.db.models.sql import Query, AggregateQuery
from django.db.models.sql.where import ExtraWhere, WhereNode
from django.utils.six import text_type, binary_type, integer_types

from .settings import ITERABLES, cachalot_settings
from .transaction import AtomicCache


class UncachableQuery(Exception):
    pass


class IsRawQuery(Exception):
    pass


CACHABLE_PARAM_TYPES = {
    bool, int, float, Decimal, bytearray, binary_type, text_type, type(None),
    datetime.date, datetime.time, datetime.datetime, datetime.timedelta, UUID,
}
CACHABLE_PARAM_TYPES.update(integer_types)  # Adds long for Python 2
UNCACHABLE_FUNCS = {Now, TransactionNow}

try:
    from psycopg2 import Binary
    from psycopg2.extras import (
        NumericRange, DateRange, DateTimeRange, DateTimeTZRange, Inet, Json)
    from django.contrib.postgres.fields.jsonb import JsonAdapter

except ImportError:
    pass
else:
    CACHABLE_PARAM_TYPES.update((
        Binary, NumericRange, DateRange, DateTimeRange, DateTimeTZRange, Inet,
        Json, JsonAdapter))


def check_parameter_types(params):
    for p in params:
        cl = p.__class__
        if cl not in CACHABLE_PARAM_TYPES:
            if cl in ITERABLES:
                check_parameter_types(p)
            elif cl is dict:
                check_parameter_types(p.items())
            else:
                raise UncachableQuery


def get_query_cache_key(compiler):
    """
    Generates a cache key from a SQLCompiler.

    This cache key is specific to the SQL query and its context
    (which database is used).  The same query in the same context
    (= the same database) must generate the same cache key.

    :arg compiler: A SQLCompiler that will generate the SQL query
    :type compiler: django.db.models.sql.compiler.SQLCompiler
    :return: A cache key
    :rtype: int
    """
    sql, params = compiler.as_sql()
    check_parameter_types(params)
    cache_key = '%s:%s:%s' % (compiler.using, sql,
                              [text_type(p) for p in params])
    return sha1(cache_key.encode('utf-8')).hexdigest()


def get_table_cache_key(db_alias, table):
    """
    Generates a cache key from a SQL table.

    :arg db_alias: Alias of the used database
    :type db_alias: str or unicode
    :arg table: Name of the SQL table
    :type table: str or unicode
    :return: A cache key
    :rtype: int
    """
    cache_key = '%s:%s' % (db_alias, table)
    return sha1(cache_key.encode('utf-8')).hexdigest()


def _get_tables_from_sql(connection, lowercased_sql):
    return {t for t in connection.introspection.django_table_names()
            if t in lowercased_sql}


def _find_subqueries_in_where(children):
    for child in children:
        child_class = child.__class__
        if child_class is WhereNode:
            for grand_child in _find_subqueries_in_where(child.children):
                yield grand_child
        elif child_class is ExtraWhere:
            raise IsRawQuery
        else:
            rhs = child.rhs
            rhs_class = rhs.__class__
            if rhs_class is Query:
                yield rhs
            elif rhs_class is QuerySet:
                yield rhs.query
            elif rhs_class is Subquery or rhs_class is Exists:
                yield rhs.queryset.query
            elif rhs_class in UNCACHABLE_FUNCS:
                raise UncachableQuery


def is_cachable(table):
    whitelist = cachalot_settings.CACHALOT_ONLY_CACHABLE_TABLES
    if whitelist and table not in whitelist:
        return False
    return table not in cachalot_settings.CACHALOT_UNCACHABLE_TABLES


def are_all_cachable(tables):
    whitelist = cachalot_settings.CACHALOT_ONLY_CACHABLE_TABLES
    if whitelist and not tables.issubset(whitelist):
        return False
    return tables.isdisjoint(cachalot_settings.CACHALOT_UNCACHABLE_TABLES)


def filter_cachable(tables):
    whitelist = cachalot_settings.CACHALOT_ONLY_CACHABLE_TABLES
    tables = tables.difference(cachalot_settings.CACHALOT_UNCACHABLE_TABLES)
    if whitelist:
        return tables.intersection(whitelist)
    return tables


def _get_tables(db_alias, query):
    if query.select_for_update or (
            not cachalot_settings.CACHALOT_CACHE_RANDOM
            and '?' in query.order_by):
        raise UncachableQuery

    try:
        if query.extra_select:
            raise IsRawQuery
        # Gets all tables already found by the ORM.
        tables = set(query.table_map)
        tables.add(query.get_meta().db_table)
        # Gets tables in subquery annotations.
        for annotation in query.annotations.values():
            if isinstance(annotation, Subquery):
                tables.update(_get_tables(db_alias, annotation.queryset.query))
        # Gets tables in WHERE subqueries.
        for subquery in _find_subqueries_in_where(query.where.children):
            tables.update(_get_tables(db_alias, subquery))
        # Gets tables in HAVING subqueries.
        if isinstance(query, AggregateQuery):
            tables.update(
                _get_tables_from_sql(connections[db_alias], query.subquery))
    except IsRawQuery:
        sql = query.get_compiler(db_alias).as_sql()[0].lower()
        tables = _get_tables_from_sql(connections[db_alias], sql)

    if not are_all_cachable(tables):
        raise UncachableQuery
    return tables


def _get_table_cache_keys(compiler):
    db_alias = compiler.using
    get_table_cache_key = cachalot_settings.CACHALOT_TABLE_KEYGEN
    return [get_table_cache_key(db_alias, t)
            for t in _get_tables(db_alias, compiler.query)]


def _invalidate_tables(cache, db_alias, tables):
    tables = filter_cachable(set(tables))
    if not tables:
        return
    now = time()
    get_table_cache_key = cachalot_settings.CACHALOT_TABLE_KEYGEN
    cache.set_many(
        {get_table_cache_key(db_alias, t): now for t in tables},
        cachalot_settings.CACHALOT_TIMEOUT)

    if isinstance(cache, AtomicCache):
        cache.to_be_invalidated.update(tables)


1			# coding: utf-8
2
3			from __future__ import unicode_literals
4			import datetime
5			from decimal import Decimal
6			from hashlib import sha1
7			from time import time
8			from uuid import UUID
9
10			from django.contrib.postgres.functions import TransactionNow
11			from django.db import connections
12			from django.db.models import QuerySet, Subquery, Exists
13			from django.db.models.functions import Now
14			from django.db.models.sql import Query, AggregateQuery
15			from django.db.models.sql.where import ExtraWhere, WhereNode
16			from django.utils.six import text_type, binary_type, integer_types
17
18			from .settings import ITERABLES, cachalot_settings
19			from .transaction import AtomicCache
20
21
22			class UncachableQuery(Exception):
23			pass
24
25
26			class IsRawQuery(Exception):
27			pass
28
29
30			CACHABLE_PARAM_TYPES = {
31			bool, int, float, Decimal, bytearray, binary_type, text_type, type(None),
32			datetime.date, datetime.time, datetime.datetime, datetime.timedelta, UUID,
33			}
34			CACHABLE_PARAM_TYPES.update(integer_types) # Adds long for Python 2
35			UNCACHABLE_FUNCS = {Now, TransactionNow}
36
37			try:
38			from psycopg2 import Binary
39			from psycopg2.extras import (
40			NumericRange, DateRange, DateTimeRange, DateTimeTZRange, Inet, Json)
41			from django.contrib.postgres.fields.jsonb import JsonAdapter
42
43			except ImportError:
44			pass
45			else:
46			CACHABLE_PARAM_TYPES.update((
47			Binary, NumericRange, DateRange, DateTimeRange, DateTimeTZRange, Inet,
48			Json, JsonAdapter))
49
50
51			def check_parameter_types(params):
52			for p in params:
53			cl = p.__class__
54			if cl not in CACHABLE_PARAM_TYPES:
55			if cl in ITERABLES:
56			check_parameter_types(p)
57			elif cl is dict:
58			check_parameter_types(p.items())
59			else:
60			raise UncachableQuery
61
62
63			def get_query_cache_key(compiler):
64			"""
65			Generates a cache key from a SQLCompiler.
66
67			This cache key is specific to the SQL query and its context
68			(which database is used). The same query in the same context
69			(= the same database) must generate the same cache key.
70
71			:arg compiler: A SQLCompiler that will generate the SQL query
72			:type compiler: django.db.models.sql.compiler.SQLCompiler
73			:return: A cache key
74			:rtype: int
75			"""
76			sql, params = compiler.as_sql()
77			check_parameter_types(params)
78			cache_key = '%s:%s:%s' % (compiler.using, sql,
79			[text_type(p) for p in params])
80			return sha1(cache_key.encode('utf-8')).hexdigest()
81
82
83			def get_table_cache_key(db_alias, table):
84			"""
85			Generates a cache key from a SQL table.
86
87			:arg db_alias: Alias of the used database
88			:type db_alias: str or unicode
89			:arg table: Name of the SQL table
90			:type table: str or unicode
91			:return: A cache key
92			:rtype: int
93			"""
94			cache_key = '%s:%s' % (db_alias, table)
95			return sha1(cache_key.encode('utf-8')).hexdigest()
96
97
98			def _get_tables_from_sql(connection, lowercased_sql):
99			return {t for t in connection.introspection.django_table_names()
100			if t in lowercased_sql}
101
102
103			def _find_subqueries_in_where(children):
104			for child in children:
105			child_class = child.__class__
106			if child_class is WhereNode:
107			for grand_child in _find_subqueries_in_where(child.children):
108			yield grand_child
109			elif child_class is ExtraWhere:
110			raise IsRawQuery
111			else:
112			rhs = child.rhs
113			rhs_class = rhs.__class__
114			if rhs_class is Query:
115			yield rhs
116			elif rhs_class is QuerySet:
117			yield rhs.query
118			elif rhs_class is Subquery or rhs_class is Exists:
119			yield rhs.queryset.query
120			elif rhs_class in UNCACHABLE_FUNCS:
121			raise UncachableQuery
122
123
124			def is_cachable(table):
125			whitelist = cachalot_settings.CACHALOT_ONLY_CACHABLE_TABLES
126			if whitelist and table not in whitelist:
127			return False
128			return table not in cachalot_settings.CACHALOT_UNCACHABLE_TABLES
129
130
131			def are_all_cachable(tables):
132			whitelist = cachalot_settings.CACHALOT_ONLY_CACHABLE_TABLES
133			if whitelist and not tables.issubset(whitelist):
134			return False
135			return tables.isdisjoint(cachalot_settings.CACHALOT_UNCACHABLE_TABLES)
136
137
138			def filter_cachable(tables):
139			whitelist = cachalot_settings.CACHALOT_ONLY_CACHABLE_TABLES
140			tables = tables.difference(cachalot_settings.CACHALOT_UNCACHABLE_TABLES)
141			if whitelist:
142			return tables.intersection(whitelist)
143			return tables
144
145
146			def _get_tables(db_alias, query):
147			if query.select_for_update or (
148			not cachalot_settings.CACHALOT_CACHE_RANDOM
149			and '?' in query.order_by):
150			raise UncachableQuery
151
152			try:
153			if query.extra_select:
154			raise IsRawQuery
155			# Gets all tables already found by the ORM.
156			tables = set(query.table_map)
157			tables.add(query.get_meta().db_table)
158			# Gets tables in subquery annotations.
159			for annotation in query.annotations.values():
160			if isinstance(annotation, Subquery):
161			tables.update(_get_tables(db_alias, annotation.queryset.query))
162			# Gets tables in WHERE subqueries.
163			for subquery in _find_subqueries_in_where(query.where.children):
164			tables.update(_get_tables(db_alias, subquery))
165			# Gets tables in HAVING subqueries.
166			if isinstance(query, AggregateQuery):
167			tables.update(
168			_get_tables_from_sql(connections[db_alias], query.subquery))
169			except IsRawQuery:
170			sql = query.get_compiler(db_alias).as_sql()[0].lower()
171			tables = _get_tables_from_sql(connections[db_alias], sql)
172
173			if not are_all_cachable(tables):
174			raise UncachableQuery
175			return tables
176
177
178			def _get_table_cache_keys(compiler):
179			db_alias = compiler.using
180			get_table_cache_key = cachalot_settings.CACHALOT_TABLE_KEYGEN
181			return [get_table_cache_key(db_alias, t)
182			for t in _get_tables(db_alias, compiler.query)]
183
184
185			def _invalidate_tables(cache, db_alias, tables):
186			tables = filter_cachable(set(tables))
187			if not tables:
188			return
189			now = time()
190			get_table_cache_key = cachalot_settings.CACHALOT_TABLE_KEYGEN
191			cache.set_many(
192			{get_table_cache_key(db_alias, t): now for t in tables},
193			cachalot_settings.CACHALOT_TIMEOUT)
194
195			if isinstance(cache, AtomicCache):
196			cache.to_be_invalidated.update(tables)
197

noripyt / django-cachalot

Push — master ( 6e429f...74e000 )

_find_subqueries_in_where() C

Complexity

Size

Duplication

Importance

How to fix Complexity

Complexity

Duplication Side-by-Side

Filter issues like