cache_template_stream() - Code Metrics - Inspection of "improve browse cache" - ergoithz/browsepy - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — 0.5.2 ( 5c22ef...024653 )

by Felipe A.

created 2016-12-07 16:47 UTC

cache_template_stream() A

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	3
c	0
b	0
f	0
dl	0
loc	18
rs	9.4285

#!/usr/bin/env python
# -*- coding: UTF-8 -*-

import logging
import os
import os.path
import json
import base64
import io
import gzip

from flask import Flask, request, render_template, redirect, send_file, \
                  url_for, send_from_directory, stream_with_context, \
                  make_response, current_app
from werkzeug.exceptions import NotFound

from .__meta__ import __app__, __version__, __license__, __author__  # noqa
from .file import Node, OutsideRemovableBase, OutsideDirectoryBase, \
                  secure_filename
from . import compat
from . import manager

__basedir__ = os.path.abspath(os.path.dirname(compat.fsdecode(__file__)))

logger = logging.getLogger(__name__)

app = Flask(
    __name__,
    static_url_path='/static',
    static_folder=os.path.join(__basedir__, 'static'),
    template_folder=os.path.join(__basedir__, 'templates')
    )
app.config.update(
    directory_base=compat.getcwd(),
    directory_start=compat.getcwd(),
    directory_remove=None,
    directory_upload=None,
    directory_tar_buffsize=262144,
    directory_downloadable=True,
    use_binary_multiples=True,
    plugin_modules=[],
    plugin_namespaces=(
        'browsepy.plugin',
        'browsepy_',
        '',
        ),
    disk_cache_enable=True,
    cache_class='browsepy.cache:SimpleLRUCache',
    cache_kwargs={'maxsize': 32},
    cache_browse_key='view/browse<{sort}>/{path}',
    browse_sort_properties=['text', 'type', 'modified', 'size']
    )
app.jinja_env.add_extension('browsepy.extensions.HTMLCompress')

if "BROWSEPY_SETTINGS" in os.environ:
    app.config.from_envvar("BROWSEPY_SETTINGS")

plugin_manager = manager.PluginManager(app)


def iter_cookie_browse_sorting():
    '''
    Get sorting-cookie data of current request.

    :yields: tuple of path and sorting property
    :ytype: 2-tuple of strings
    '''
    try:
        data = request.cookies.get('browse-sorting', 'e30=').encode('ascii')
        valid = current_app.config.get('browse_sort_properties', ())
        for path, prop in json.loads(base64.b64decode(data).decode('utf-8')):
            if prop.startswith('-'):
                if prop[1:] in valid:
                    yield path, prop
            elif prop in valid:
                yield path, prop
    except (ValueError, TypeError, KeyError) as e:
        logger.exception(e)


def get_cookie_browse_sorting(path, default):
    '''
    Get sorting-cookie data for path of current request.

    :returns: sorting property
    :rtype: string
    '''
    for cpath, cprop in iter_cookie_browse_sorting():
        if path == cpath:
            return cprop
    return default


def browse_sortkey_reverse(prop):
    '''
    Get sorting function for browse

    :returns: tuple with sorting gunction and reverse bool
    :rtype: tuple of a dict and a bool
    '''
    if prop.startswith('-'):
        prop = prop[1:]
        reverse = True
    else:
        reverse = False

    if prop == 'text':
        return (
            lambda x: (
                x.is_directory == reverse,
                x.link.text.lower() if x.link and x.link.text else x.name
                ),
            reverse
            )
    if prop == 'size':
        return (
            lambda x: (
                x.is_directory == reverse,
                x.stats.st_size
                ),
            reverse
            )
    return (
        lambda x: (
            x.is_directory == reverse,
            getattr(x, prop, None)
            ),
        reverse
        )


def cache_template_stream(key, stream):
    '''
    Yield and cache jinja template stream.

    :param key: cache key
    :type key: str
    :param stream: jinja template stream
    :type stream: iterable
    :yields: rendered jinja template chunks
    :ytype: str
    '''
    buffer = io.BytesIO()
    with gzip.GzipFile(mode='wb', fileobj=buffer) as f:
        for part in stream:
            yield part
            f.write(part.encode('utf-8'))
    cache = current_app.extensions['plugin_manager'].cache
    cache.set(key, (buffer.getvalue(), request.url_root))


def stream_template(template_name, **context):
    '''
    Some templates can be huge, this function returns an streaming response,
    sending the content in chunks and preventing from timeout.

    :param template_name: template
    :param **context: parameters for templates.
    :yields: HTML strings
    '''
    cache_key = context.pop('cache_key', None)
    app.update_template_context(context)
    stream = app.jinja_env.get_template(template_name).generate(context)
    if cache_key:
        stream = cache_template_stream(cache_key, stream)
    return current_app.response_class(stream_with_context(stream))


def get_cached_response(key):
    '''
    Get cached response object from key.

    :param key: cache key
    :type key: str
    :return: response object
    :rtype: flask.Response
    '''
    cache = current_app.extensions['plugin_manager'].cache
    data, url_root = cache.get(key) or (None, None)
    if data and url_root == request.url_root:
        if 'gzip' in request.headers.get('Accept-Encoding', '').lower():
            response = current_app.response_class(data)
            response.headers['Content-Encoding'] = 'gzip'
            response.headers['Vary'] = 'Accept-Encoding'
            response.headers['Content-Length'] = len(data)
            return response
        return send_file(
            gzip.GzipFile(mode='rb', fileobj=io.BytesIO(data)),
            mimetype='text/html',
            as_attachment=False
            )


@app.context_processor
def template_globals():
    return {
        'manager': app.extensions['plugin_manager'],
        'len': len,
        }


@app.route('/sort/<string:property>', defaults={"path": ""})
@app.route('/sort/<string:property>/<path:path>')
def sort(property, path):
    try:
        directory = Node.from_urlpath(path)
    except OutsideDirectoryBase:
        return NotFound()

    if not directory.is_directory:
        return NotFound()

    data = [
        (cpath, cprop)
        for cpath, cprop in iter_cookie_browse_sorting()
        if cpath != path
        ]
    data.append((path, property))
    raw_data = base64.b64encode(json.dumps(data).encode('utf-8'))

    # prevent cookie becoming too large
    while len(raw_data) > 3975:  # 4000 - len('browse-sorting=""; Path=/')
        data.pop(0)
        raw_data = base64.b64encode(json.dumps(data).encode('utf-8'))

    response = redirect(url_for(".browse", path=directory.urlpath))
    response.set_cookie('browse-sorting', raw_data)
    return response


@app.route("/browse", defaults={"path": ""})
@app.route('/browse/<path:path>')
def browse(path):
    sort_property = get_cookie_browse_sorting(path, 'text')

    if current_app.config['disk_cache_enable']:
        cache_key = current_app.config.get('cache_browse_key', '').format(
            sort=sort_property,
            path=path
            )
        cached_response = get_cached_response(cache_key)
        if cached_response:
            return cached_response
    else:
        cache_key = None  # disables response cache

    sort_fnc, sort_reverse = browse_sortkey_reverse(sort_property)

    try:
        directory = Node.from_urlpath(path)
        if directory.is_directory:
            return stream_template(
                'browse.html',
                cache_key=cache_key,
                file=directory,
                sort_property=sort_property,
                sort_fnc=sort_fnc,
                sort_reverse=sort_reverse
                )
    except OutsideDirectoryBase:
        pass
    return NotFound()


@app.route('/open/<path:path>', endpoint="open")
def open_file(path):
    try:
        file = Node.from_urlpath(path)
        if file.is_file:
            return send_from_directory(file.parent.path, file.name)
    except OutsideDirectoryBase:
        pass
    return NotFound()


@app.route("/download/file/<path:path>")
def download_file(path):
    try:
        file = Node.from_urlpath(path)
        if file.is_file:
            return file.download()
    except OutsideDirectoryBase:
        pass
    return NotFound()


@app.route("/download/directory/<path:path>.tgz")
def download_directory(path):
    try:
        directory = Node.from_urlpath(path)
        if directory.is_directory:
            return directory.download()
    except OutsideDirectoryBase:
        pass
    return NotFound()


@app.route("/remove/<path:path>", methods=("GET", "POST"))
def remove(path):
    try:
        file = Node.from_urlpath(path)
    except OutsideDirectoryBase:
        return NotFound()
    if request.method == 'GET':
        if not file.can_remove:
            return NotFound()
        return render_template('remove.html', file=file)
    parent = file.parent
    if parent is None:
        # base is not removable
        return NotFound()

    try:
        file.remove()
    except OutsideRemovableBase:
        return NotFound()

    return redirect(url_for(".browse", path=parent.urlpath))


@app.route("/upload", defaults={'path': ''}, methods=("POST",))
@app.route("/upload/<path:path>", methods=("POST",))
def upload(path):
    try:
        directory = Node.from_urlpath(path)
    except OutsideDirectoryBase:
        return NotFound()

    if not directory.is_directory or not directory.can_upload:
        return NotFound()

    for v in request.files.listvalues():
        for f in v:
            filename = secure_filename(f.filename)
            if filename:
                filename = directory.choose_filename(filename)
                filepath = os.path.join(directory.path, filename)
                f.save(filepath)
    return redirect(url_for(".browse", path=directory.urlpath))


@app.route("/")
def index():
    path = app.config["directory_start"] or app.config["directory_base"]
    try:
        urlpath = Node(path).urlpath
    except OutsideDirectoryBase:
        return NotFound()
    return browse(urlpath)


@app.after_request
def page_not_found(response):
    if response.status_code == 404:
        return make_response((render_template('404.html'), 404))
    return response


@app.errorhandler(404)
def page_not_found_error(e):
    return render_template('404.html'), 404


@app.errorhandler(500)
def internal_server_error(e):  # pragma: no cover
    logger.exception(e)
    return getattr(e, 'message', 'Internal server error'), 500


1			#!/usr/bin/env python
2			# -- coding: UTF-8 --
3
4			import logging
5			import os
6			import os.path
7			import json
8			import base64
9			import io
10			import gzip
11
12			from flask import Flask, request, render_template, redirect, send_file, \
13			url_for, send_from_directory, stream_with_context, \
14			make_response, current_app
15			from werkzeug.exceptions import NotFound
16
17			from .__meta__ import __app__, __version__, __license__, __author__ # noqa
18			from .file import Node, OutsideRemovableBase, OutsideDirectoryBase, \
19			secure_filename
20			from . import compat
21			from . import manager
22
23			__basedir__ = os.path.abspath(os.path.dirname(compat.fsdecode(__file__)))
24
25			logger = logging.getLogger(__name__)
26
27			app = Flask(
28			__name__,
29			static_url_path='/static',
30			static_folder=os.path.join(__basedir__, 'static'),
31			template_folder=os.path.join(__basedir__, 'templates')
32			)
33			app.config.update(
34			directory_base=compat.getcwd(),
35			directory_start=compat.getcwd(),
36			directory_remove=None,
37			directory_upload=None,
38			directory_tar_buffsize=262144,
39			directory_downloadable=True,
40			use_binary_multiples=True,
41			plugin_modules=[],
42			plugin_namespaces=(
43			'browsepy.plugin',
44			'browsepy_',
45			'',
46			),
47			disk_cache_enable=True,
48			cache_class='browsepy.cache:SimpleLRUCache',
49			cache_kwargs={'maxsize': 32},
50			cache_browse_key='view/browse<{sort}>/{path}',
51			browse_sort_properties=['text', 'type', 'modified', 'size']
52			)
53			app.jinja_env.add_extension('browsepy.extensions.HTMLCompress')
54
55			if "BROWSEPY_SETTINGS" in os.environ:
56			app.config.from_envvar("BROWSEPY_SETTINGS")
57
58			plugin_manager = manager.PluginManager(app)
59
60
61			def iter_cookie_browse_sorting():
62			'''
63			Get sorting-cookie data of current request.
64
65			:yields: tuple of path and sorting property
66			:ytype: 2-tuple of strings
67			'''
68			try:
69			data = request.cookies.get('browse-sorting', 'e30=').encode('ascii')
70			valid = current_app.config.get('browse_sort_properties', ())
71			for path, prop in json.loads(base64.b64decode(data).decode('utf-8')):
72			if prop.startswith('-'):
73			if prop[1:] in valid:
74			yield path, prop
75			elif prop in valid:
76			yield path, prop
77			except (ValueError, TypeError, KeyError) as e:
78			logger.exception(e)
79
80
81			def get_cookie_browse_sorting(path, default):
82			'''
83			Get sorting-cookie data for path of current request.
84
85			:returns: sorting property
86			:rtype: string
87			'''
88			for cpath, cprop in iter_cookie_browse_sorting():
89			if path == cpath:
90			return cprop
91			return default
92
93
94			def browse_sortkey_reverse(prop):
95			'''
96			Get sorting function for browse
97
98			:returns: tuple with sorting gunction and reverse bool
99			:rtype: tuple of a dict and a bool
100			'''
101			if prop.startswith('-'):
102			prop = prop[1:]
103			reverse = True
104			else:
105			reverse = False
106
107			if prop == 'text':
108			return (
109			lambda x: (
110			x.is_directory == reverse,
111			x.link.text.lower() if x.link and x.link.text else x.name
112			),
113			reverse
114			)
115			if prop == 'size':
116			return (
117			lambda x: (
118			x.is_directory == reverse,
119			x.stats.st_size
120			),
121			reverse
122			)
123			return (
124			lambda x: (
125			x.is_directory == reverse,
126			getattr(x, prop, None)
127			),
128			reverse
129			)
130
131
132			def cache_template_stream(key, stream):
133			'''
134			Yield and cache jinja template stream.
135
136			:param key: cache key
137			:type key: str
138			:param stream: jinja template stream
139			:type stream: iterable
140			:yields: rendered jinja template chunks
141			:ytype: str
142			'''
143			buffer = io.BytesIO()
144			with gzip.GzipFile(mode='wb', fileobj=buffer) as f:
145			for part in stream:
146			yield part
147			f.write(part.encode('utf-8'))
148			cache = current_app.extensions['plugin_manager'].cache
149			cache.set(key, (buffer.getvalue(), request.url_root))
150
151
152			def stream_template(template_name, **context):
153			'''
154			Some templates can be huge, this function returns an streaming response,
155			sending the content in chunks and preventing from timeout.
156
157			:param template_name: template
158			:param **context: parameters for templates.
159			:yields: HTML strings
160			'''
161			cache_key = context.pop('cache_key', None)
162			app.update_template_context(context)
163			stream = app.jinja_env.get_template(template_name).generate(context)
164			if cache_key:
165			stream = cache_template_stream(cache_key, stream)
166			return current_app.response_class(stream_with_context(stream))
167
168
169			def get_cached_response(key):
170			'''
171			Get cached response object from key.
172
173			:param key: cache key
174			:type key: str
175			:return: response object
176			:rtype: flask.Response
177			'''
178			cache = current_app.extensions['plugin_manager'].cache
179			data, url_root = cache.get(key) or (None, None)
180			if data and url_root == request.url_root:
181			if 'gzip' in request.headers.get('Accept-Encoding', '').lower():
182			response = current_app.response_class(data)
183			response.headers['Content-Encoding'] = 'gzip'
184			response.headers['Vary'] = 'Accept-Encoding'
185			response.headers['Content-Length'] = len(data)
186			return response
187			return send_file(
188			gzip.GzipFile(mode='rb', fileobj=io.BytesIO(data)),
189			mimetype='text/html',
190			as_attachment=False
191			)
192
193
194			@app.context_processor
195			def template_globals():
196			return {
197			'manager': app.extensions['plugin_manager'],
198			'len': len,
199			}
200
201
202			@app.route('/sort/<string:property>', defaults={"path": ""})
203			@app.route('/sort/<string:property>/<path:path>')
204			def sort(property, path):
205			try:
206			directory = Node.from_urlpath(path)
207			except OutsideDirectoryBase:
208			return NotFound()
209
210			if not directory.is_directory:
211			return NotFound()
212
213			data = [
214			(cpath, cprop)
215			for cpath, cprop in iter_cookie_browse_sorting()
216			if cpath != path
217			]
218			data.append((path, property))
219			raw_data = base64.b64encode(json.dumps(data).encode('utf-8'))
220
221			# prevent cookie becoming too large
222			while len(raw_data) > 3975: # 4000 - len('browse-sorting=""; Path=/')
223			data.pop(0)
224			raw_data = base64.b64encode(json.dumps(data).encode('utf-8'))
225
226			response = redirect(url_for(".browse", path=directory.urlpath))
227			response.set_cookie('browse-sorting', raw_data)
228			return response
229
230
231			@app.route("/browse", defaults={"path": ""})
232			@app.route('/browse/<path:path>')
233			def browse(path):
234			sort_property = get_cookie_browse_sorting(path, 'text')
235
236			if current_app.config['disk_cache_enable']:
237			cache_key = current_app.config.get('cache_browse_key', '').format(
238			sort=sort_property,
239			path=path
240			)
241			cached_response = get_cached_response(cache_key)
242			if cached_response:
243			return cached_response
244			else:
245			cache_key = None # disables response cache
246
247			sort_fnc, sort_reverse = browse_sortkey_reverse(sort_property)
248
249			try:
250			directory = Node.from_urlpath(path)
251			if directory.is_directory:
252			return stream_template(
253			'browse.html',
254			cache_key=cache_key,
255			file=directory,
256			sort_property=sort_property,
257			sort_fnc=sort_fnc,
258			sort_reverse=sort_reverse
259			)
260			except OutsideDirectoryBase:
261			pass
262			return NotFound()
263
264
265			@app.route('/open/<path:path>', endpoint="open")
266			def open_file(path):
267			try:
268			file = Node.from_urlpath(path)
269			if file.is_file:
270			return send_from_directory(file.parent.path, file.name)
271			except OutsideDirectoryBase:
272			pass
273			return NotFound()
274
275
276			@app.route("/download/file/<path:path>")
277			def download_file(path):
278			try:
279			file = Node.from_urlpath(path)
280			if file.is_file:
281			return file.download()
282			except OutsideDirectoryBase:
283			pass
284			return NotFound()
285
286
287			@app.route("/download/directory/<path:path>.tgz")
288			def download_directory(path):
289			try:
290			directory = Node.from_urlpath(path)
291			if directory.is_directory:
292			return directory.download()
293			except OutsideDirectoryBase:
294			pass
295			return NotFound()
296
297
298			@app.route("/remove/<path:path>", methods=("GET", "POST"))
299			def remove(path):
300			try:
301			file = Node.from_urlpath(path)
302			except OutsideDirectoryBase:
303			return NotFound()
304			if request.method == 'GET':
305			if not file.can_remove:
306			return NotFound()
307			return render_template('remove.html', file=file)
308			parent = file.parent
309			if parent is None:
310			# base is not removable
311			return NotFound()
312
313			try:
314			file.remove()
315			except OutsideRemovableBase:
316			return NotFound()
317
318			return redirect(url_for(".browse", path=parent.urlpath))
319
320
321			@app.route("/upload", defaults={'path': ''}, methods=("POST",))
322			@app.route("/upload/<path:path>", methods=("POST",))
323			def upload(path):
324			try:
325			directory = Node.from_urlpath(path)
326			except OutsideDirectoryBase:
327			return NotFound()
328
329			if not directory.is_directory or not directory.can_upload:
330			return NotFound()
331
332			for v in request.files.listvalues():
333			for f in v:
334			filename = secure_filename(f.filename)
335			if filename:
336			filename = directory.choose_filename(filename)
337			filepath = os.path.join(directory.path, filename)
338			f.save(filepath)
339			return redirect(url_for(".browse", path=directory.urlpath))
340
341
342			@app.route("/")
343			def index():
344			path = app.config["directory_start"] or app.config["directory_base"]
345			try:
346			urlpath = Node(path).urlpath
347			except OutsideDirectoryBase:
348			return NotFound()
349			return browse(urlpath)
350
351
352			@app.after_request
353			def page_not_found(response):
354			if response.status_code == 404:
355			return make_response((render_template('404.html'), 404))
356			return response
357
358
359			@app.errorhandler(404)
360			def page_not_found_error(e):
361			return render_template('404.html'), 404
362
363
364			@app.errorhandler(500)
365			def internal_server_error(e): # pragma: no cover
366			logger.exception(e)
367			return getattr(e, 'message', 'Internal server error'), 500
368

ergoithz / browsepy

Push — 0.5.2 ( 5c22ef...024653 )

cache_template_stream() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like