Thumbnails refactor

Add timestamp signature validation for non-authenticated users

Document thumbnailing

Add signing snippet for full image urls

Always sign private image urls

Sign thumbnail urls with url path

Always sign generate thumbnail urls and use data from signature

Remove django-sendfile

Include README files in source in documentation using m2r

Rename info variable to sig_info
parent ba5fa057
......@@ -5,7 +5,6 @@ name = "pypi"
[packages]
django-localflavor = "*"
django-sendfile = "*"
freezegun = "*"
bleach = "*"
"django-tinymce4-lite" = "*"
......@@ -26,6 +25,7 @@ uWSGI = "*"
"django-bootstrap4" = "*"
firebase-admin = "*"
sentry-sdk = "*"
django-sendfile2 = "*"
[dev-packages]
django-template-check = "*"
......
This diff is collapsed.
utils.media package
===================
.. automodule:: utils.media
:members:
:undoc-members:
:show-inheritance:
Submodules
----------
utils.media.services module
---------------------------
.. automodule:: utils.media.services
:members:
:undoc-members:
:show-inheritance:
utils.media.views module
------------------------
.. automodule:: utils.media.views
:members:
:undoc-members:
:show-inheritance:
......@@ -13,6 +13,7 @@ Subpackages
utils.conscribo
utils.management
utils.media
utils.templatetags
Submodules
......@@ -66,12 +67,4 @@ utils.validators module
:undoc-members:
:show-inheritance:
utils.views module
------------------
.. automodule:: utils.views
:members:
:undoc-members:
:show-inheritance:
......@@ -5,7 +5,7 @@ from django.utils.translation import ugettext_lazy as _
from members.templatetags.member_card import member_card
from thaliawebsite.templatetags.grid_item import grid_item
from utils.templatetags.thumbnail import thumbnail
from utils.media.services import get_thumbnail_url
register = template.Library()
......@@ -14,7 +14,8 @@ register = template.Library()
def membergroup_card(group):
image_url = static('activemembers/images/placeholder_overview.png')
if group.photo:
image_url = thumbnail(group.photo, settings.THUMBNAIL_SIZES['medium'])
image_url = get_thumbnail_url(group.photo,
settings.THUMBNAIL_SIZES['medium'])
return grid_item(
title=group.name,
......
......@@ -5,7 +5,7 @@ from django.urls import reverse
from django.utils.translation import ugettext_lazy as _
from thaliawebsite.templatetags.grid_item import grid_item
from utils.templatetags.thumbnail import thumbnail
from utils.media.services import get_thumbnail_url
register = template.Library()
......@@ -18,8 +18,8 @@ def member_card(member, meta_text=None, ribbon=None):
image_url = static('members/images/default-avatar.jpg')
if member.profile.photo:
image_url = thumbnail(member.profile.photo,
settings.THUMBNAIL_SIZES['medium'])
image_url = get_thumbnail_url(member.profile.photo,
settings.THUMBNAIL_SIZES['medium'])
return grid_item(
title=member.profile.display_name(),
......
......@@ -4,7 +4,7 @@ from django.template.defaultfilters import striptags, truncatechars
from thaliawebsite.templatetags.bleach_tags import bleach
from thaliawebsite.templatetags.grid_item import grid_item
from utils.templatetags.thumbnail import thumbnail
from utils.media.services import get_thumbnail_url
register = template.Library()
......@@ -13,8 +13,9 @@ register = template.Library()
def partner_card(partner):
image_url = ''
if partner.logo:
image_url = thumbnail(partner.logo, settings.THUMBNAIL_SIZES['medium'],
fit=False)
image_url = get_thumbnail_url(partner.logo,
settings.THUMBNAIL_SIZES['medium'],
fit=False)
meta_text = truncatechars(bleach(striptags(partner.company_profile)), 80)
......@@ -30,11 +31,12 @@ def partner_card(partner):
@register.inclusion_tag('includes/grid_item.html')
def partner_image_card(image):
class_name = 'partner-image-card'
image_url = thumbnail(image, settings.THUMBNAIL_SIZES['medium'])
image_url = get_thumbnail_url(image, settings.THUMBNAIL_SIZES['medium'])
return grid_item(
title='',
url=thumbnail(image, settings.THUMBNAIL_SIZES['large'], fit=False),
url=get_thumbnail_url(image, settings.THUMBNAIL_SIZES['large'],
fit=False),
image_url=image_url,
class_name=class_name,
anchor_attrs='data-fancybox="gallery"'
......@@ -45,9 +47,9 @@ def partner_image_card(image):
def vacancy_card(vacancy):
image_url = None
if vacancy.get_company_logo():
image_url = thumbnail(vacancy.get_company_logo(),
settings.THUMBNAIL_SIZES['medium'],
fit=False)
image_url = get_thumbnail_url(vacancy.get_company_logo(),
settings.THUMBNAIL_SIZES['medium'],
fit=False)
description = truncatechars(bleach(striptags(vacancy.description)), 150)
extra_class = 'external-vacancy'
......
......@@ -13,7 +13,7 @@ class PhotoRetrieveSerializer(serializers.ModelSerializer):
if obj:
file = obj.file
return create_image_thumbnail_dict(
self.context['request'], file)
self.context['request'], file, fit_large=False)
class Meta:
model = Photo
......
......@@ -2,10 +2,9 @@ from django import template
from django.conf import settings
from django.template.defaultfilters import date
from django.urls import reverse
from photos.templatetags.shared_thumbnail import shared_thumbnail
from thaliawebsite.templatetags.grid_item import grid_item
from utils.templatetags.thumbnail import thumbnail
from utils.media.services import get_thumbnail_url
register = template.Library()
......@@ -16,8 +15,8 @@ def album_card(album):
image_url = ''
if album.cover:
image_url = thumbnail(album.cover.file,
settings.THUMBNAIL_SIZES['medium'])
image_url = get_thumbnail_url(album.cover.file,
settings.THUMBNAIL_SIZES['medium'])
if album.cover.rotation > 0:
class_name += ' rotate{}'.format(album.cover.rotation)
......@@ -47,19 +46,20 @@ def photo_card(photo):
anchor_attrs += ' data-download={}'.format(
reverse('photos:download', args=[photo.album.slug, photo]))
image_url = thumbnail(photo.file, settings.THUMBNAIL_SIZES['medium'])
image_url = get_thumbnail_url(photo.file,
settings.THUMBNAIL_SIZES['medium'])
if photo.album.shareable:
image_url = shared_thumbnail(photo.album.slug,
photo.album.access_token, photo.file,
settings.THUMBNAIL_SIZES['medium'])
image_url = get_thumbnail_url(photo.file,
settings.THUMBNAIL_SIZES['medium'])
if photo.rotation > 0:
class_name += ' rotate{}'.format(photo.rotation)
return grid_item(
title='',
url=thumbnail(photo.file, settings.THUMBNAIL_SIZES['large'],
fit=False),
url=get_thumbnail_url(photo.file,
settings.THUMBNAIL_SIZES['large'],
fit=False),
image_url=image_url,
class_name=class_name,
anchor_attrs=anchor_attrs
......
from django import template
from django.urls import resolve, reverse
from os.path import basename
from utils.templatetags.thumbnail import thumbnail
register = template.Library()
@register.simple_tag
def shared_thumbnail(slug, token, path, size, fit=True):
thumb = resolve(thumbnail(path, size, fit))
filename = basename(thumb.kwargs['path'])
args = [slug, thumb.kwargs['size_fit'], token, filename]
return reverse('photos:shared-thumbnail', args=args)
......@@ -16,11 +16,6 @@ urlpatterns = [
path('<filename>', views.shared_download, name='shared-download'),
])),
])),
path('thumbnail/', include([
re_path(r'(?P<size_fit>\d+x\d+_[01])/', include([
path('<token>/<filename>/', views.shared_thumbnail, name='shared-thumbnail'),
])),
])),
path('<token>/', views.shared_album, name='shared-album'),
])),
]
......@@ -13,7 +13,6 @@ from photos.models import Album, Photo
from photos.services import (check_shared_album_token,
get_annotated_accessible_albums,
is_album_accessible)
from utils.views import _private_thumbnails_unauthed
COVER_FILENAME = 'cover.jpg'
......@@ -142,10 +141,3 @@ def shared_album_download(request, slug, token):
album = get_object_or_404(Album, slug=slug)
check_shared_album_token(album, token)
return _album_download(request, album)
def shared_thumbnail(request, slug, size_fit, token, filename):
album = get_object_or_404(Album, slug=slug)
check_shared_album_token(album, token)
photopath = _photo_path(album, filename)
return _private_thumbnails_unauthed(request, size_fit, photopath)
......@@ -2,7 +2,7 @@ from django import template
from django.urls import reverse
from thaliawebsite.templatetags.grid_item import grid_item
from utils.templatetags.thumbnail import thumbnail
from utils.media.services import get_thumbnail_url
register = template.Library()
......@@ -32,6 +32,6 @@ def thabloid_card(year, thabloid):
),
meta_text=buttons,
url=None,
image_url=thumbnail(thabloid.cover, '255x360'),
image_url=get_thumbnail_url(thabloid.cover, '255x360'),
class_name=f'thabloid-card mix {class_name} col-6 col-md-3 my-3',
)
from django.conf import settings
from utils.templatetags.thumbnail import thumbnail
from utils.media.services import get_media_url, get_thumbnail_url
def create_image_thumbnail_dict(request, file, placeholder='',
size_small=settings.THUMBNAIL_SIZES['small'],
size_medium=settings.THUMBNAIL_SIZES['medium'],
size_large=settings.THUMBNAIL_SIZES['large']):
size_large=settings.THUMBNAIL_SIZES['large'],
fit_small=True, fit_medium=True,
fit_large=True):
if file:
return {
'full': request.build_absolute_uri('{}{}'.format(
settings.MEDIA_URL, file)),
'small': request.build_absolute_uri(thumbnail(
file, size_small, 1, True)),
'medium': request.build_absolute_uri(thumbnail(
file, size_medium, 1, True)),
'large': request.build_absolute_uri(thumbnail(
file, size_large, 1, True))
'full': request.build_absolute_uri(
get_media_url(file)),
'small': request.build_absolute_uri(get_thumbnail_url(
file, size_small, fit=fit_small)),
'medium': request.build_absolute_uri(get_thumbnail_url(
file, size_medium, fit=fit_medium)),
'large': request.build_absolute_uri(get_thumbnail_url(
file, size_large, fit=fit_large))
}
return {
'full': placeholder,
......
......@@ -40,18 +40,16 @@ from django.views.generic import TemplateView
from django.views.i18n import JavaScriptCatalog
import members
from members.sitemaps import sitemap as members_sitemap
from members.views import ObtainThaliaAuthToken
from activemembers.sitemaps import sitemap as activemembers_sitemap
from documents.sitemaps import sitemap as documents_sitemap
from events.sitemaps import sitemap as events_sitemap
from events.views import AlumniEventsView
from members.sitemaps import sitemap as members_sitemap
from members.views import ObtainThaliaAuthToken
from partners.sitemaps import sitemap as partners_sitemap
from thabloid.sitemaps import sitemap as thabloid_sitemap
from thaliawebsite.forms import AuthenticationForm
from utils.views import (generate_thumbnail, private_thumbnails,
private_thumbnails_api)
from utils.media.views import (generate_thumbnail, private_media)
from . import views
from .sitemaps import StaticViewSitemap
......@@ -98,8 +96,6 @@ urlpatterns = [ # pylint: disable=invalid-name
])),
url(r'^career/', include('partners.urls')),
url(r'^contact$', TemplateView.as_view(template_name='singlepages/contact.html'), name='contact'),
url(r'^private-thumbnails/(?P<size_fit>\d+x\d+_[01])/(?P<path>.*)', private_thumbnails, name='private-thumbnails'),
url(r'^generate-thumbnail/(?P<size_fit>\d+x\d+_[01])/(?P<path>[^/]+)/(?P<thumbpath>[^/]+)', generate_thumbnail, name='generate-thumbnail'),
url(r'^api/', include([
url(r'wikilogin', views.wiki_login),
url(r'^v1/', include([
......@@ -112,8 +108,6 @@ urlpatterns = [ # pylint: disable=invalid-name
url(r'^', include('pizzas.api.urls')),
url(r'^', include('photos.api.urls')),
url(r'^', include('pushnotifications.api.urls')),
url(r'^generate-thumbnail/(?P<size_fit>\d+x\d+_[01])/(?P<path>[^/]+)/(?P<thumbpath>[^/]+)', generate_thumbnail, {'api': True}, name='generate-thumbnail-api'),
url(r'^private-thumbnails/(?P<size_fit>\d+x\d+_[01])/(?P<path>.*)', private_thumbnails_api, name='private-thumbnails-api'),
])),
])),
url(r'^education/', include('education.urls')),
......@@ -133,5 +127,8 @@ urlpatterns = [ # pylint: disable=invalid-name
url(r'jsi18n/$', JavaScriptCatalog.as_view(), name='javascript-catalog'),
# Provide something to test error handling. Limited to admins.
url(r'crash/$', views.crash),
# Custom media paths
url(r'^media/(public|private)/generate-thumbnail/(?P<request_path>.*)', generate_thumbnail, name='generate-thumbnail'),
url(r'^media/private/(?P<request_path>.*)$', private_media, name='private-media')
] + static(settings.MEDIA_URL + 'public/',
document_root=os.path.join(settings.MEDIA_ROOT, 'public'))
Media & Thumbnailing
=====
This document explains how the `utils.media` package enables us to serve public
and private media (user-uploaded) files. We also give an outline of the
workings of our thumbnailing implementation.
## Media types
We differentiate between two types of media: public and private. Public means
that the files can be served without any kind of authentication. Requests for
private files have to be checked by Django first before we offload serving
the file using [django-sendfile2](https://github.com/moggers87/django-sendfile2).
### Public
The public files are saved in `MEDIA_ROOT/public/`.
These files can be served by nginx like one would with a regular
[`MEDIA_ROOT`](https://docs.djangoproject.com/en/2.1/ref/settings/#media-root).
The files should be served at `MEDIA_URL/public/`.
### Private
The private files are saved in `MEDIA_ROOT` where the `public` folder is
the exception. This is a legacy implementation from our previous thumbnailing
code. We could decide to migrate these files to a dedicated `private` folder
in the future. Even though the files are not saved in such a folder they
are located at `MEDIA_URL/private/`. This path should be available to concrexit
so that the media files can be served correctly: the signature
(more on that below) should be valid and match the path.
#### Signatures
To make sure the private media files stay private we have implemented a
signature-based security mechanism based on the idea behind the
[Thumbor](https://thumbor.readthedocs.io/en/latest/security.html)'s security
implementation. This signature can be extended to contain more information,
like we did for our thumbnails.
The value of the signature, which is a dictionary, must at least contain
one key: `serve_path`. This is the path that is used in `utils.media.views.private_media()`
to determine the location of the media file that should be served. If this path
matches the path in the url the file will be made available to the user.
A second, optional, key is `attachment` which is used by the sendfile backend
to force a download if the value is `True`.
```python
sig_info = { 'serve_path': f'{settings.MEDIA_ROOT}/<image location>' }
print(signing.dumps(sig_info))
'eyJzZXJ2ZV9wYXRoIjoiL21lZGlhLzxpbWFnZSBsb2NhdGlvbj4ifQ:1gsbnC:QJTqFUWY6HxMBTEIxYPl9V1yf48'
```
The signature is appended to the location using a query parameter with
the key `sig` and is valid for 3 hours as implemented by
`utils.media.views._get_image_information`.
```
https://<base url>/media/private/<image location>?sig=eyJzZXJ2ZV9wYXRoIjoiL21lZGlhLzx...
```
To get the url of a media item you can use `utils.media.services.get_media_url()`. **Never use this to get a url directly from user input!**
## Thumbnails
To make sure we do not have to serve full-size images to users every time they
open a page we decided to thumbnail our images. This functionality is provided
as a templatetag (`utils.templatetags.thumbnail`) and service (`utils.media.services.get_thumbnail_url()`).
Every thumbnail only needs to be generated once, when the thumbnail exists the
tools mentioned above will return the direct media url relative to the root of
the concrexit instance. If the image is private the url will contain the
signature as well.
If the thumbnail does not yet exist the returned url will be the url of the
generation function (`utils.media.views.generate_thumbnail`). This url can then
be called by the browser of the user creating the thumbnails on-demand preventing
blocking of the page or large workloads when uploading multiple photos at once.
Once the thumbnail is generated the user will be redirected to the real image.
The url to the thumbnail generation route is signed with a signature that
extends the signature we use to serve private media files. More information
about the signature can be found in the next section.
**The thumbnailing tools should never be used to create thumbnails from user
input directly!** They do not protect against directory traversel and assume
the path is correct. They are meant to be used with a path from an `ImageField`
or similar.
```mermaid
graph TB
get[get_thumbnail_url]
generate[generate_thumbnail]
public[serve thumbnail as public media file]
private[serve thumbnail as private media file]
get-->|no thumbnail|generate
get-->|thumbnail exists\n& is public|public
generate-->|redirects|public
generate-->|redirects|private
get-->|thumbnail exists\n& is private|private
```
### Signatures
The signature used for the generation of thumbnails extends the signature to
load a private media file with keys used for the generation. The signature
contains all information required to generate the thumbnail.
```python
{
'visibility': 'public',
'size': '300x300',
'fit': 0,
'path': 'image.jpeg',
'thumb_path': 'thumbnails/300x300_0/image.jpeg',
'serve_path': '/media/public/thumbnails/300x300_0/image.jpeg'
}
```
The signature protects us against path tampering and thus path traversal and
DDoS attacks. We also know that a user can only get a valid signature if
they accessed a page providing them with that signature.
import os
from django.db.models.fields.files import ImageFieldFile
from django.conf import settings
from django.core import signing
from django.urls import reverse
def get_media_url(path, attachment=False):
"""
Get the url of the provided media file to serve in a browser.
If the file is private a signature will be added.
Do NOT use this with user input
:param path: the location of the file
:param attachment: True if the file is a forced download
:return: the url of the media
"""
if isinstance(path, ImageFieldFile):
path = path.name
parts = path.split('/')
query = ''
url_path = path
sig_info = {
'visibility': 'private' if parts[0] != 'public' else 'public',
'serve_path': os.path.join(settings.MEDIA_ROOT, path),
'attachment': attachment
}
if sig_info['visibility'] == 'private':
# Add private to path and calculate signature
url_path = f'private/{path}'
query = f'?sig={signing.dumps(sig_info)}'
return f'{settings.MEDIA_URL}{url_path}{query}'
def get_thumbnail_url(path, size, fit=True):
"""
Get the thumbnail url of a media file. NEVER use this with user input.
If the thumbnail exists this function will return the url of the
media file, with signature if necessary. Does it not yet exist a route
that executes the :func:`utils.media.views.generate_thumbnail`
will be the output.
:param path: the location of the file
:param size: size of the image
:param fit: False to keep the aspect ratio, True to crop
:return: direct media url or generate-thumbnail path
"""
if isinstance(path, ImageFieldFile):
path = path.name
query = ''
size_fit = '{}_{}'.format(size, int(fit))
parts = path.split('/')
sig_info = {
'size': size,
'fit': int(fit),
'path': path,
}
# Check if the image is public and assemble useful information
if parts[0] == 'public':
sig_info['path'] = '/'.join(parts[1:])
sig_info['visibility'] = 'public'
else:
sig_info['visibility'] = 'private'
sig_info['thumb_path'] = f'thumbnails/{size_fit}/{sig_info["path"]}'
url_path = (f'{sig_info["visibility"]}/thumbnails/'
f'{size_fit}/{sig_info["path"]}')
if sig_info['visibility'] == 'public':
full_original_path = os.path.join(
settings.MEDIA_ROOT, 'public', sig_info['path'])
full_thumb_path = os.path.join(
settings.MEDIA_ROOT, 'public', sig_info['thumb_path'])
else:
full_original_path = os.path.join(
settings.MEDIA_ROOT, sig_info['path'])
full_thumb_path = os.path.join(
settings.MEDIA_ROOT, sig_info['thumb_path'])
sig_info['serve_path'] = full_thumb_path
# Check if we need to generate, then redirect to the generating route,
# otherwise just return the serving file path
if (not os.path.isfile(full_thumb_path) or
os.path.getmtime(full_original_path)
> os.path.getmtime(full_thumb_path)):
# Put all image info in signature for the generate view
query = f'?sig={signing.dumps(sig_info)}'
# We provide a URL instead of calling it as a function, so that using
# it means kicking off a new GET request. If we would generate all
# thumbnails inline, loading an album overview would have high latency.
return reverse('generate-thumbnail',
args=[sig_info['visibility'],
os.path.join(size_fit, sig_info["path"])]) + query
if sig_info['visibility'] == 'private':
# Put all image info in signature for serve view
query = f'?sig={signing.dumps(sig_info)}'
return f'{settings.MEDIA_URL}{url_path}{query}'
"""Utility views"""
import os
from datetime import timedelta
from PIL import Image, ImageOps
from django.conf import settings
from django.core import signing
from django.core.exceptions import PermissionDenied
from django.core.signing import BadSignature
from django.http import Http404
from django.shortcuts import redirect
from sendfile import sendfile
def _get_signature_info(request):
if 'sig' in request.GET:
signature = request.GET.get('sig')
try:
return signing.loads(signature, max_age=timedelta(hours=3))
except BadSignature:
pass
raise PermissionDenied
def private_media(request, request_path):
"""
Serve private media files
:param request: the request
:return: the media file
"""
# Get image information from signature
# raises PermissionDenied if bad signature
info = _get_signature_info(request)
if (not os.path.isfile(info['serve_path'])
or not info['serve_path'].endswith(request_path)):
# 404 if the file does not exist
raise Http404("Media not found.")
# Serve the file
return sendfile(request, info['serve_path'],
attachment=info.get('attachment', False))