1
|
|
|
# coding: utf-8 |
2
|
|
|
|
3
|
|
|
from __future__ import unicode_literals |
4
|
|
|
import datetime |
5
|
|
|
from decimal import Decimal |
6
|
|
|
from hashlib import sha1 |
7
|
|
|
from time import time |
8
|
|
|
from uuid import UUID |
9
|
|
|
|
10
|
|
|
from django import VERSION as django_version |
11
|
|
|
from django.db import connections |
12
|
|
|
from django.db.models.expressions import RawSQL |
13
|
|
|
from django.db.models.sql import Query |
14
|
|
|
from django.db.models.sql.where import ( |
15
|
|
|
ExtraWhere, SubqueryConstraint, WhereNode) |
16
|
|
|
from django.utils.module_loading import import_string |
17
|
|
|
from django.utils.six import text_type, binary_type |
18
|
|
|
|
19
|
|
|
from .settings import cachalot_settings |
20
|
|
|
from .transaction import AtomicCache |
21
|
|
|
|
22
|
|
|
|
23
|
|
|
class UncachableQuery(Exception): |
24
|
|
|
pass |
25
|
|
|
|
26
|
|
|
|
27
|
|
|
class IsRawQuery(Exception): |
28
|
|
|
pass |
29
|
|
|
|
30
|
|
|
|
31
|
|
|
TUPLE_OR_LIST = {tuple, list} |
32
|
|
|
|
33
|
|
|
CACHABLE_PARAM_TYPES = { |
34
|
|
|
bool, int, float, Decimal, bytearray, binary_type, text_type, type(None), |
35
|
|
|
datetime.date, datetime.time, datetime.datetime, datetime.timedelta, UUID, |
36
|
|
|
} |
37
|
|
|
|
38
|
|
|
UNCACHABLE_FUNCS = set() |
39
|
|
|
if django_version[:2] >= (1, 9): |
40
|
|
|
from django.db.models.functions import Now |
41
|
|
|
from django.contrib.postgres.functions import TransactionNow |
42
|
|
|
UNCACHABLE_FUNCS.update((Now, TransactionNow)) |
43
|
|
|
|
44
|
|
|
try: |
45
|
|
|
from psycopg2 import Binary |
46
|
|
|
from psycopg2.extras import ( |
47
|
|
|
NumericRange, DateRange, DateTimeRange, DateTimeTZRange, Inet, Json) |
48
|
|
|
except ImportError: |
49
|
|
|
pass |
50
|
|
|
else: |
51
|
|
|
CACHABLE_PARAM_TYPES.update(( |
52
|
|
|
Binary, |
53
|
|
|
NumericRange, DateRange, DateTimeRange, DateTimeTZRange, Inet, Json)) |
54
|
|
|
if django_version[:2] >= (1, 11): |
55
|
|
|
from django.contrib.postgres.fields.jsonb import JsonAdapter |
56
|
|
|
CACHABLE_PARAM_TYPES.add(JsonAdapter) |
57
|
|
|
|
58
|
|
|
|
59
|
|
|
def check_parameter_types(params): |
60
|
|
|
for p in params: |
61
|
|
|
cl = p.__class__ |
62
|
|
|
if cl not in CACHABLE_PARAM_TYPES: |
63
|
|
|
if cl in TUPLE_OR_LIST: |
64
|
|
|
check_parameter_types(p) |
65
|
|
|
elif cl is dict: |
66
|
|
|
check_parameter_types(p.items()) |
67
|
|
|
else: |
68
|
|
|
raise UncachableQuery |
69
|
|
|
|
70
|
|
|
|
71
|
|
|
def get_query_cache_key(compiler): |
72
|
|
|
""" |
73
|
|
|
Generates a cache key from a SQLCompiler. |
74
|
|
|
|
75
|
|
|
This cache key is specific to the SQL query and its context |
76
|
|
|
(which database is used). The same query in the same context |
77
|
|
|
(= the same database) must generate the same cache key. |
78
|
|
|
|
79
|
|
|
:arg compiler: A SQLCompiler that will generate the SQL query |
80
|
|
|
:type compiler: django.db.models.sql.compiler.SQLCompiler |
81
|
|
|
:return: A cache key |
82
|
|
|
:rtype: int |
83
|
|
|
""" |
84
|
|
|
sql, params = compiler.as_sql() |
85
|
|
|
check_parameter_types(params) |
86
|
|
|
cache_key = '%s:%s:%s' % (compiler.using, sql, |
87
|
|
|
[text_type(p) for p in params]) |
88
|
|
|
return sha1(cache_key.encode('utf-8')).hexdigest() |
89
|
|
|
|
90
|
|
|
|
91
|
|
|
def get_table_cache_key(db_alias, table): |
92
|
|
|
""" |
93
|
|
|
Generates a cache key from a SQL table. |
94
|
|
|
|
95
|
|
|
:arg db_alias: Alias of the used database |
96
|
|
|
:type db_alias: str or unicode |
97
|
|
|
:arg table: Name of the SQL table |
98
|
|
|
:type table: str or unicode |
99
|
|
|
:return: A cache key |
100
|
|
|
:rtype: int |
101
|
|
|
""" |
102
|
|
|
cache_key = '%s:%s' % (db_alias, table) |
103
|
|
|
return sha1(cache_key.encode('utf-8')).hexdigest() |
104
|
|
|
|
105
|
|
|
|
106
|
|
|
def _get_query_cache_key(compiler): |
107
|
|
|
return import_string(cachalot_settings.CACHALOT_QUERY_KEYGEN)(compiler) |
108
|
|
|
|
109
|
|
|
|
110
|
|
|
def _get_table_cache_key(db_alias, table): |
111
|
|
|
return import_string(cachalot_settings.CACHALOT_TABLE_KEYGEN)(db_alias, table) |
112
|
|
|
|
113
|
|
|
|
114
|
|
|
def _get_tables_from_sql(connection, lowercased_sql): |
115
|
|
|
return {t for t in connection.introspection.django_table_names() |
116
|
|
|
if t in lowercased_sql} |
117
|
|
|
|
118
|
|
|
|
119
|
|
|
def _find_subqueries(children): |
120
|
|
|
for child in children: |
121
|
|
|
child_class = child.__class__ |
122
|
|
|
if child_class is WhereNode: |
123
|
|
|
for grand_child in _find_subqueries(child.children): |
124
|
|
|
yield grand_child |
125
|
|
|
elif child_class is SubqueryConstraint: |
126
|
|
|
if child.query_object.__class__ is Query: |
127
|
|
|
yield child.query_object |
128
|
|
|
else: |
129
|
|
|
yield child.query_object.query |
130
|
|
|
elif child_class is ExtraWhere: |
131
|
|
|
raise IsRawQuery |
132
|
|
|
else: |
133
|
|
|
rhs = None |
134
|
|
|
if hasattr(child, 'rhs'): |
135
|
|
|
rhs = child.rhs |
136
|
|
|
rhs_class = rhs.__class__ |
137
|
|
|
if rhs_class is RawSQL: |
138
|
|
|
raise IsRawQuery |
139
|
|
|
if rhs_class is Query: |
140
|
|
|
yield rhs |
141
|
|
|
elif hasattr(rhs, 'query'): |
142
|
|
|
yield rhs.query |
143
|
|
|
elif rhs_class in UNCACHABLE_FUNCS: |
144
|
|
|
raise UncachableQuery |
145
|
|
|
|
146
|
|
|
|
147
|
|
|
def is_cachable(table): |
148
|
|
|
whitelist = cachalot_settings.CACHALOT_ONLY_CACHABLE_TABLES |
149
|
|
|
if whitelist and table not in whitelist: |
150
|
|
|
return False |
151
|
|
|
return table not in cachalot_settings.CACHALOT_UNCACHABLE_TABLES |
152
|
|
|
|
153
|
|
|
|
154
|
|
|
def are_all_cachable(tables): |
155
|
|
|
whitelist = cachalot_settings.CACHALOT_ONLY_CACHABLE_TABLES |
156
|
|
|
if whitelist and not tables.issubset(whitelist): |
157
|
|
|
return False |
158
|
|
|
return tables.isdisjoint(cachalot_settings.CACHALOT_UNCACHABLE_TABLES) |
159
|
|
|
|
160
|
|
|
|
161
|
|
|
def filter_cachable(tables): |
162
|
|
|
whitelist = cachalot_settings.CACHALOT_ONLY_CACHABLE_TABLES |
163
|
|
|
tables = tables.difference(cachalot_settings.CACHALOT_UNCACHABLE_TABLES) |
164
|
|
|
if whitelist: |
165
|
|
|
return tables.intersection(whitelist) |
166
|
|
|
return tables |
167
|
|
|
|
168
|
|
|
|
169
|
|
|
def _get_tables(db_alias, query): |
170
|
|
|
if query.select_for_update or ( |
171
|
|
|
'?' in query.order_by |
172
|
|
|
and not cachalot_settings.CACHALOT_CACHE_RANDOM): |
173
|
|
|
raise UncachableQuery |
174
|
|
|
|
175
|
|
|
try: |
176
|
|
|
if query.extra_select or getattr(query, 'subquery', False): |
177
|
|
|
raise IsRawQuery |
178
|
|
|
tables = set(query.table_map) |
179
|
|
|
tables.add(query.get_meta().db_table) |
180
|
|
|
for subquery in _find_subqueries(query.where.children): |
181
|
|
|
tables.update(_get_tables(db_alias, subquery)) |
182
|
|
|
except IsRawQuery: |
183
|
|
|
sql = query.get_compiler(db_alias).as_sql()[0].lower() |
184
|
|
|
tables = _get_tables_from_sql(connections[db_alias], sql) |
185
|
|
|
|
186
|
|
|
if not are_all_cachable(tables): |
187
|
|
|
raise UncachableQuery |
188
|
|
|
return tables |
189
|
|
|
|
190
|
|
|
|
191
|
|
|
def _get_table_cache_keys(compiler): |
192
|
|
|
db_alias = compiler.using |
193
|
|
|
return [_get_table_cache_key(db_alias, t) |
194
|
|
|
for t in _get_tables(db_alias, compiler.query)] |
195
|
|
|
|
196
|
|
|
|
197
|
|
|
def _invalidate_tables(cache, db_alias, tables): |
198
|
|
|
tables = filter_cachable(set(tables)) |
199
|
|
|
if not tables: |
200
|
|
|
return |
201
|
|
|
now = time() |
202
|
|
|
cache.set_many( |
203
|
|
|
{_get_table_cache_key(db_alias, t): now for t in tables}, |
204
|
|
|
cachalot_settings.CACHALOT_TIMEOUT) |
205
|
|
|
|
206
|
|
|
if isinstance(cache, AtomicCache): |
207
|
|
|
cache.to_be_invalidated.update(tables) |
208
|
|
|
|