Add Celery to the thaliawebsite setup

Dockerfile

@@ -54,8 +54,10 @@ RUN if [ "$install_dev_requirements" -eq 1 ]; then \
 # Create entry points
 COPY resources/entrypoint.sh /usr/local/bin/entrypoint.sh
 COPY resources/entrypoint_production.sh /usr/local/bin/entrypoint_production.sh
+COPY resources/entrypoint_celery.sh /usr/local/bin/entrypoint_celery.sh
 RUN chmod +x /usr/local/bin/entrypoint.sh && \
-    chmod +x /usr/local/bin/entrypoint_production.sh
+    chmod +x /usr/local/bin/entrypoint_production.sh && \
+    chmod +x /usr/local/bin/entrypoint_celery.sh
 
 # copy app source
 COPY website /usr/src/app/website/

docker-compose.yml

@@ -6,6 +6,11 @@ services:
             - /var/lib/postgresql/
         environment: &postgresvars
             POSTGRES_DB: thalia
+    redis:
+        image: redis:4.0.9
+        command: redis-server --appendonly yes
+        volumes:
+          - redis:/data
     web:
         image: registry.gitlab.com/thaliawww/concrexit
         build: .
@@ -17,10 +22,27 @@ services:
         volumes:
             - ./website:/usr/src/app/website/
             - concrexit:/concrexit/
-        environment:
+        environment: &webvars
             <<: *postgresvars
             DJANGO_DEBUG: 'True'
             DJANGO_POSTGRES_HOST: postgres
+            CELERY_REDIS_HOST: redis
+            YOLO_ENV: 'tralala'
+    celery:
+        image: registry.gitlab.com/thaliawww/concrexit
+        build: .
+        entrypoint: /usr/local/bin/entrypoint_celery.sh
+        volumes_from:
+          - web
+        links:
+          - postgres
+        depends_on:
+          - redis
+        environment:
+            <<: *webvars
+
 volumes:
     concrexit:
         driver: local
+    redis:
+        driver: local

docs/thaliawebsite.rst

@@ -26,6 +26,14 @@ thaliawebsite\.admin module
     :undoc-members:
     :show-inheritance:
 
+thaliawebsite\.celery module
+----------------------------
+
+.. automodule:: thaliawebsite.celery
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 thaliawebsite\.forms module
 ---------------------------
 

requirements.txt

@@ -13,3 +13,6 @@ django-libsass>=0.7,<1
 django-cors-headers>=2.1.0,<2.2
 python-magic>=0.4.13,<0.5
 pyfcm>=1.4.2,<1.5
+celery>=4.0<4.2
+redis>=2.10<2.11
+django-celery-results>=1.0.1<1.1

resources/entrypoint_celery.sh

+#!/bin/bash
+
+set -e
+
+# Wait for Redis server to start
+sleep 10
+
+# Could do the following if redis-tools installed:
+# X="`redis-cli -h \"$CELERY_BROKER_HOST\" ping`"
+# echo ${X}
+#
+# while  [ "${X}" != "PONG" ]; do
+#     >&2 echo "Redis is unavailable: Sleeping"
+#     echo "${X}"
+#     sleep 5
+# done
+# >&2 echo "Redis is up"
+
+cd /usr/src/app/website/
+>&2 echo "Starting celery worker"
+celery worker -A thaliawebsite

website/thaliawebsite/__init__.py

+from __future__ import absolute_import, unicode_literals
+
+# This will make sure the app is always imported when
+# Django starts so that shared_task will use this app.
+from .celery import app as celery_app
+
+__all__ = ['celery_app']

website/thaliawebsite/celery.py

+from __future__ import absolute_import, unicode_literals
+import os
+from celery import Celery
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'thaliawebsite.settings')
+
+app = Celery('thaliawebsite')
+
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object('django.conf:settings', namespace='CELERY')
+
+# Load task modules from all registered Django app configs.
+app.autodiscover_tasks()

website/thaliawebsite/settings/production.py

@@ -125,6 +125,11 @@ if os.environ.get('DJANGO_EMAIL_HOST'):
     EMAIL_USE_SSL = os.environ.get('DJANGO_EMAIL_USE_SSL', False) == 'True'
     EMAIL_TIMEOUT = 10
 
+# Celery settings
+CELERY_BROKER_URL = 'redis://{}:6379/0'.format(
+    os.environ.get('CELERY_REDIS_HOST')
+)
+
 # Secure headers
 X_FRAME_OPTIONS = 'DENY'
 SECURE_CONTENT_TYPE_NOSNIFF = True

website/thaliawebsite/settings/settings.py

@@ -54,6 +54,7 @@ INSTALLED_APPS = [
     'rest_framework.authtoken',
     'compressor',
     'corsheaders',
+    'django_celery_results',
     # Our apps
     # Directly link to the app config when applicable as recommended
     # by the docs: https://docs.djangoproject.com/en/2.0/ref/applications/
@@ -240,6 +241,13 @@ COMPRESS_CSS_FILTERS = ['compressor.filters.css_default.CssAbsoluteFilter',
 # Precompiler settings
 STATIC_PRECOMPILER_LIST_FILES = True
 
+# Celery settings
+CELERY_BROKER_URL = 'redis://localhost:6379/0'
+# Checkout caveats for the timeout config below:
+# http://docs.celeryproject.org/en/latest/getting-started/brokers/redis.html#id1
+CELERY_BROKER_TRANSPORT_OPTIONS = {'visibility_timeout': 15778800}
+CELERY_RESULT_BACKEND = 'django-db'
+
 # Membership prices
 MEMBERSHIP_PRICES = {
     'year': 7.5,