|
1
|
|
|
# coding: utf-8 |
|
2
|
|
|
|
|
3
|
|
|
from __future__ import unicode_literals |
|
4
|
|
|
import datetime |
|
5
|
|
|
from decimal import Decimal |
|
6
|
|
|
from hashlib import sha1 |
|
7
|
|
|
from time import time |
|
8
|
|
|
from uuid import UUID |
|
9
|
|
|
|
|
10
|
|
|
from django import VERSION as django_version |
|
11
|
|
|
from django.db import connections |
|
12
|
|
|
from django.db.models.sql import Query |
|
13
|
|
|
from django.db.models.sql.where import ExtraWhere, SubqueryConstraint |
|
14
|
|
|
from django.utils.module_loading import import_string |
|
15
|
|
|
from django.utils.six import text_type, binary_type |
|
16
|
|
|
|
|
17
|
|
|
from .settings import cachalot_settings |
|
18
|
|
|
from .signals import post_invalidation |
|
19
|
|
|
from .transaction import AtomicCache |
|
20
|
|
|
|
|
21
|
|
|
|
|
22
|
|
|
DJANGO_LTE_1_8 = django_version <= (1, 8) |
|
23
|
|
|
|
|
24
|
|
|
|
|
25
|
|
|
class UncachableQuery(Exception): |
|
26
|
|
|
pass |
|
27
|
|
|
|
|
28
|
|
|
|
|
29
|
|
|
CACHABLE_PARAM_TYPES = { |
|
30
|
|
|
bool, int, float, Decimal, binary_type, text_type, type(None), |
|
31
|
|
|
datetime.date, datetime.time, datetime.datetime, datetime.timedelta, UUID, |
|
32
|
|
|
} |
|
33
|
|
|
|
|
34
|
|
|
try: |
|
35
|
|
|
from psycopg2.extras import ( |
|
36
|
|
|
NumericRange, DateRange, DateTimeRange, DateTimeTZRange, Inet) |
|
37
|
|
|
except ImportError: |
|
38
|
|
|
pass |
|
39
|
|
|
else: |
|
40
|
|
|
CACHABLE_PARAM_TYPES.update(( |
|
41
|
|
|
NumericRange, DateRange, DateTimeRange, DateTimeTZRange, Inet)) |
|
42
|
|
|
|
|
43
|
|
|
|
|
44
|
|
|
def check_parameter_types(params): |
|
45
|
|
|
for p in params: |
|
46
|
|
|
cl = p.__class__ |
|
47
|
|
|
if cl not in CACHABLE_PARAM_TYPES: |
|
48
|
|
|
if cl is list or cl is tuple: |
|
49
|
|
|
check_parameter_types(p) |
|
50
|
|
|
elif cl is dict: |
|
51
|
|
|
check_parameter_types(p.items()) |
|
52
|
|
|
else: |
|
53
|
|
|
raise UncachableQuery |
|
54
|
|
|
|
|
55
|
|
|
|
|
56
|
|
|
def get_query_cache_key(compiler): |
|
57
|
|
|
""" |
|
58
|
|
|
Generates a cache key from a SQLCompiler. |
|
59
|
|
|
|
|
60
|
|
|
This cache key is specific to the SQL query and its context |
|
61
|
|
|
(which database is used). The same query in the same context |
|
62
|
|
|
(= the same database) must generate the same cache key. |
|
63
|
|
|
|
|
64
|
|
|
:arg compiler: A SQLCompiler that will generate the SQL query |
|
65
|
|
|
:type compiler: django.db.models.sql.compiler.SQLCompiler |
|
66
|
|
|
:return: A cache key |
|
67
|
|
|
:rtype: str |
|
68
|
|
|
""" |
|
69
|
|
|
sql, params = compiler.as_sql() |
|
70
|
|
|
check_parameter_types(params) |
|
71
|
|
|
cache_key = '%s:%s:%s' % (compiler.using, sql, params) |
|
72
|
|
|
return sha1(cache_key.encode('utf-8')).hexdigest() |
|
73
|
|
|
|
|
74
|
|
|
|
|
75
|
|
|
def get_table_cache_key(db_alias, table): |
|
76
|
|
|
""" |
|
77
|
|
|
Generates a cache key from a SQL table. |
|
78
|
|
|
|
|
79
|
|
|
:arg db_alias: Alias of the used database |
|
80
|
|
|
:type db_alias: str or unicode |
|
81
|
|
|
:arg table: Name of the SQL table |
|
82
|
|
|
:type table: str or unicode |
|
83
|
|
|
:return: A cache key |
|
84
|
|
|
:rtype: str |
|
85
|
|
|
""" |
|
86
|
|
|
cache_key = '%s:%s' % (db_alias, table) |
|
87
|
|
|
return sha1(cache_key.encode('utf-8')).hexdigest() |
|
88
|
|
|
|
|
89
|
|
|
|
|
90
|
|
|
def _get_query_cache_key(compiler): |
|
91
|
|
|
return import_string(cachalot_settings.CACHALOT_QUERY_KEYGEN)(compiler) |
|
92
|
|
|
|
|
93
|
|
|
|
|
94
|
|
|
def _get_table_cache_key(db_alias, table): |
|
95
|
|
|
return import_string(cachalot_settings.CACHALOT_TABLE_KEYGEN)(db_alias, table) |
|
96
|
|
|
|
|
97
|
|
|
|
|
98
|
|
|
def _get_tables_from_sql(connection, lowercased_sql): |
|
99
|
|
|
return [t for t in connection.introspection.django_table_names() |
|
100
|
|
|
if t in lowercased_sql] |
|
101
|
|
|
|
|
102
|
|
|
|
|
103
|
|
|
def _find_subqueries(children): |
|
104
|
|
|
for child in children: |
|
105
|
|
|
if child.__class__ is SubqueryConstraint: |
|
106
|
|
|
if child.query_object.__class__ is Query: |
|
107
|
|
|
yield child.query_object |
|
108
|
|
|
else: |
|
109
|
|
|
yield child.query_object.query |
|
110
|
|
|
else: |
|
111
|
|
|
rhs = None |
|
112
|
|
|
if hasattr(child, 'rhs'): |
|
113
|
|
|
rhs = child.rhs |
|
114
|
|
|
elif child.__class__ is tuple: |
|
115
|
|
|
rhs = child[-1] |
|
116
|
|
|
if rhs.__class__ is Query: |
|
117
|
|
|
yield rhs |
|
118
|
|
|
elif hasattr(rhs, 'query'): |
|
119
|
|
|
yield rhs.query |
|
120
|
|
|
if hasattr(child, 'children'): |
|
121
|
|
|
for grand_child in _find_subqueries(child.children): |
|
122
|
|
|
yield grand_child |
|
123
|
|
|
|
|
124
|
|
|
|
|
125
|
|
|
def _get_tables(query, db_alias): |
|
126
|
|
|
if '?' in query.order_by and not cachalot_settings.CACHALOT_CACHE_RANDOM: |
|
127
|
|
|
raise UncachableQuery |
|
128
|
|
|
|
|
129
|
|
|
tables = set(query.table_map) |
|
130
|
|
|
tables.add(query.get_meta().db_table) |
|
131
|
|
|
children = query.where.children |
|
132
|
|
|
if DJANGO_LTE_1_8: |
|
133
|
|
|
children += query.having.children |
|
134
|
|
|
subquery_constraints = _find_subqueries(children) |
|
135
|
|
|
for subquery in subquery_constraints: |
|
136
|
|
|
tables.update(_get_tables(subquery, db_alias)) |
|
137
|
|
|
if query.extra_select or hasattr(query, 'subquery') \ |
|
138
|
|
|
or any(c.__class__ is ExtraWhere for c in query.where.children): |
|
139
|
|
|
sql = query.get_compiler(db_alias).as_sql()[0].lower() |
|
140
|
|
|
additional_tables = _get_tables_from_sql(connections[db_alias], sql) |
|
141
|
|
|
tables.update(additional_tables) |
|
142
|
|
|
|
|
143
|
|
|
whitelist = cachalot_settings.CACHALOT_ONLY_CACHABLE_TABLES |
|
144
|
|
|
blacklist = cachalot_settings.CACHALOT_UNCACHABLE_TABLES |
|
145
|
|
|
if (whitelist and not tables.issubset(whitelist)) \ |
|
146
|
|
|
or not tables.isdisjoint(blacklist): |
|
147
|
|
|
raise UncachableQuery |
|
148
|
|
|
return tables |
|
149
|
|
|
|
|
150
|
|
|
|
|
151
|
|
|
def _get_table_cache_keys(compiler): |
|
152
|
|
|
db_alias = compiler.using |
|
153
|
|
|
tables = _get_tables(compiler.query, db_alias) |
|
154
|
|
|
return [_get_table_cache_key(db_alias, t) for t in tables] |
|
155
|
|
|
|
|
156
|
|
|
|
|
157
|
|
|
def _invalidate_tables(cache, db_alias, tables): |
|
158
|
|
|
now = time() |
|
159
|
|
|
d = {} |
|
160
|
|
|
for table in tables: |
|
161
|
|
|
d[_get_table_cache_key(db_alias, table)] = now |
|
162
|
|
|
cache.set_many(d, None) |
|
163
|
|
|
|
|
164
|
|
|
if isinstance(cache, AtomicCache): |
|
165
|
|
|
cache.to_be_invalidated.update(tables) |
|
166
|
|
|
|
|
167
|
|
|
|
|
168
|
|
|
def _invalidate_table(cache, db_alias, table): |
|
169
|
|
|
cache.set(_get_table_cache_key(db_alias, table), time(), None) |
|
170
|
|
|
|
|
171
|
|
|
if isinstance(cache, AtomicCache): |
|
172
|
|
|
cache.to_be_invalidated.add(table) |
|
173
|
|
|
else: |
|
174
|
|
|
post_invalidation.send(table, db_alias=db_alias) |
|
175
|
|
|
|
noripyt /
django-cachalot
