Verified Commit 188bbcc7 authored by Sébastiaan Versteeg's avatar Sébastiaan Versteeg
Browse files

Remove Celery dependency

parent 78af8b13
......@@ -56,10 +56,8 @@ RUN pip install --no-cache-dir \
# Create entry points
COPY resources/entrypoint.sh /usr/local/bin/entrypoint.sh
COPY resources/entrypoint_production.sh /usr/local/bin/entrypoint_production.sh
COPY resources/entrypoint_celery.sh /usr/local/bin/entrypoint_celery.sh
RUN chmod +x /usr/local/bin/entrypoint.sh && \
chmod +x /usr/local/bin/entrypoint_production.sh && \
chmod +x /usr/local/bin/entrypoint_celery.sh
chmod +x /usr/local/bin/entrypoint_production.sh
# copy app source
COPY website /usr/src/app/website/
......
......@@ -15,13 +15,11 @@ django-ical = "*"
django-libsass = "*"
django-cors-headers = "*"
python-magic = "*"
celery = "*"
redis = "*"
raven = "*"
Django = ">=2.1,<2.2"
Pillow = "*"
django_compressor = "*"
django_celery_results = "*"
"psycopg2-binary" = "*"
bcrypt = "*"
"argon2_cffi" = "*"
......
{
"_meta": {
"hash": {
"sha256": "578ce12347a823b00a53ce5996d4778ba357b368eaf677a6b767ebdfa79d7f80"
"sha256": "24422ed4d4ab21978c31bc550d14ef87328366a5ebaf3903efd91ddbe34af434"
},
"pipfile-spec": 6,
"requires": {},
......@@ -14,13 +14,6 @@
]
},
"default": {
"amqp": {
"hashes": [
"sha256:073dd02fdd73041bffc913b767866015147b61f2a9bc104daef172fc1a0066eb",
"sha256:eed41946890cd43e8dee44a316b85cf6fee5a1a34bb4a562b660a358eb529e1b"
],
"version": "==2.3.2"
},
"argon2-cffi": {
"hashes": [
"sha256:003f588de43a817af6ecc1c06103fa0801de63849db3cb0f37576bb2da29043d",
......@@ -84,12 +77,6 @@
"index": "pypi",
"version": "==3.1.4"
},
"billiard": {
"hashes": [
"sha256:ed65448da5877b5558f19d2f7f11f8355ea76b3e63e1c0a6059f47cfae5f1c84"
],
"version": "==3.5.0.4"
},
"bleach": {
"hashes": [
"sha256:48d39675b80a75f6d1c3bdbffec791cf0bbbab665cf01e20da701c77de278718",
......@@ -111,14 +98,6 @@
],
"version": "==3.0.0"
},
"celery": {
"hashes": [
"sha256:77dab4677e24dc654d42dfbdfed65fa760455b6bb563a0877ecc35f4cfcfc678",
"sha256:ad7a7411772b80a4d6c64f2f7f723200e39fb66cf614a7fdfab76d345acc7b13"
],
"index": "pypi",
"version": "==4.2.1"
},
"certifi": {
"hashes": [
"sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c",
......@@ -193,13 +172,6 @@
"index": "pypi",
"version": "==0.0.7"
},
"django-celery-results": {
"hashes": [
"sha256:8bca2605eeff4418be7ce428a6958d64bee0f5bdf1f8e563fbc09a9e2f3d990f",
"sha256:dfa240fb535a1a2d01c9e605ad71629909318eae6b893c5009eafd7265fde10b"
],
"version": "==1.0.1"
},
"django-compressor": {
"hashes": [
"sha256:7732676cfb9d58498dfb522b036f75f3f253f72ea1345ac036434fdc418c2e57",
......@@ -278,17 +250,17 @@
},
"google-api-core": {
"hashes": [
"sha256:35c0fdb7d0ea8e699d46611c31cdbbaef55c4b3905b394f16a8c41800be5de51",
"sha256:7cf597628cb9c5ceb24834b30a325dc271d3ba15d868d81c20aa80a77e13be65"
"sha256:383993eba1036c942f0d87497bac646b55ad8b4337d41527ce50e640768d769a",
"sha256:d32b2b9d31ef799b4ec5bcd35541ec2b3254e0d86a5606f2720d4ff2c62b4527"
],
"version": "==1.5.1"
"version": "==1.5.2"
},
"google-auth": {
"hashes": [
"sha256:9ca363facbf2622d9ba828017536ccca2e0f58bd15e659b52f312172f8815530",
"sha256:a4cf9e803f2176b5de442763bd339b313d3f1ed3002e3e1eb6eec1d7c9bbc9b4"
"sha256:1cf96ac82fd37877e54c16264ebbb98b472de411ef87a4626cfc0b5b5df16d43",
"sha256:afd71b88f3ba873d86fe52aa2d352915ed579ca6b9fa3d3750c2e55656ae5aff"
],
"version": "==1.5.1"
"version": "==1.6.0"
},
"google-cloud-core": {
"hashes": [
......@@ -381,13 +353,6 @@
],
"version": "==2.2.2"
},
"kombu": {
"hashes": [
"sha256:86adec6c60f63124e2082ea8481bbe4ebe04fde8ebed32c177c7f0cd2c1c9082",
"sha256:b274db3a4eacc4789aeb24e1de3e460586db7c4fc8610f7adcc7a3a1709a60af"
],
"version": "==4.2.1"
},
"libsass": {
"hashes": [
"sha256:2313f0e82de034eea59443c8f69420c60c55f7c07fd3b59ea7e7a108b36e9d86",
......@@ -486,39 +451,39 @@
},
"psycopg2-binary": {
"hashes": [
"sha256:04afb59bbbd2eab3148e6816beddc74348078b8c02a1113ea7f7822f5be4afe3",
"sha256:098b18f4d8857a8f9b206d1dc54db56c2255d5d26458917e7bcad61ebfe4338f",
"sha256:0bf855d4a7083e20ead961fda4923887094eaeace0ab2d76eb4aa300f4bbf5bd",
"sha256:197dda3ffd02057820be83fe4d84529ea70bf39a9a4daee1d20ffc74eb3d042e",
"sha256:278ef63afb4b3d842b4609f2c05ffbfb76795cf6a184deeb8707cd5ed3c981a5",
"sha256:3cbf8c4fc8f22f0817220891cf405831559f4d4c12c4f73913730a2ea6c47a47",
"sha256:4305aed922c4d9d6163ab3a41d80b5a1cfab54917467da8168552c42cad84d32",
"sha256:47ee296f704fb8b2a616dec691cdcfd5fa0f11943955e88faa98cbd1dc3b3e3d",
"sha256:4a0e38cb30457e70580903367161173d4a7d1381eb2f2cfe4e69b7806623f484",
"sha256:4d6c294c6638a71cafb82a37f182f24321f1163b08b5d5ca076e11fe838a3086",
"sha256:4f3233c366500730f839f92833194fd8f9a5c4529c8cd8040aa162c3740de8e5",
"sha256:5221f5a3f4ca2ddf0d58e8b8a32ca50948be9a43351fda797eb4e72d7a7aa34d",
"sha256:5c6ca0b507540a11eaf9e77dee4f07c131c2ec80ca0cffa146671bf690bc1c02",
"sha256:789bd89d71d704db2b3d5e67d6d518b158985d791d3b2dec5ab85457cfc9677b",
"sha256:7b94d29239efeaa6a967f3b5971bd0518d2a24edd1511edbf4a2c8b815220d07",
"sha256:89bc65ef3301c74cf32db25334421ea6adbe8f65601ea45dcaaf095abed910bb",
"sha256:89d6d3a549f405c20c9ae4dc94d7ed2de2fa77427a470674490a622070732e62",
"sha256:97521704ac7127d7d8ba22877da3c7bf4a40366587d238ec679ff38e33177498",
"sha256:a395b62d5f44ff6f633231abe568e2203b8fabf9797cd6386aa92497df912d9a",
"sha256:a6d32c37f714c3f34158f3fa659f3a8f2658d5f53c4297d45579b9677cc4d852",
"sha256:a89ee5c26f72f2d0d74b991ce49e42ddeb4ac0dc2d8c06a0f2770a1ab48f4fe0",
"sha256:b4c8b0ef3608e59317bfc501df84a61e48b5445d45f24d0391a24802de5f2d84",
"sha256:b5fcf07140219a1f71e18486b8dc28e2e1b76a441c19374805c617aa6d9a9d55",
"sha256:b86f527f00956ecebad6ab3bb30e3a75fedf1160a8716978dd8ce7adddedd86f",
"sha256:be4c4aa22ba22f70de36c98b06480e2f1697972d49eb20d525f400d204a6d272",
"sha256:c2ac7aa1a144d4e0e613ac7286dae85671e99fe7a1353954d4905629c36b811c",
"sha256:de26ef4787b5e778e8223913a3e50368b44e7480f83c76df1f51d23bd21cea16",
"sha256:e70ebcfc5372dc7b699c0110454fc4263967f30c55454397e5769eb72c0eb0ce",
"sha256:eadbd32b6bc48b67b0457fccc94c86f7ccc8178ab839f684eb285bb592dc143e",
"sha256:ecbc6dfff6db06b8b72ae8a2f25ff20fbdcb83cb543811a08f7cb555042aa729"
"sha256:0d3ca99a80d6018d3ee9723c4d7e5cbe04d20032cdb49469e7db5c8945951ac8",
"sha256:0f8a48ad5957bf420c3cc891417f1a41d363f33c00248d6c320d3910a2f7ab07",
"sha256:3187730658f8be85c0e761b49e7a3763507ebadd2c68be78a0153ef43a261c2d",
"sha256:35ecda69ea4ceb84e59dff47a6755b07069e45e1b18e50bccdd258da0bd42350",
"sha256:4bac49711e8b6b023600571259b0792795d00518a0f32df68ca2c4f8d7cda8ea",
"sha256:5f693cb5aa2ac94a62de3e45b2b0cba76c975daadeb4eaa26c64d302fcb7ca6c",
"sha256:6804e5d1cdba2e84dce38353e827ae3b2262f2c2f784675e18802ee6252cc289",
"sha256:6edd27d913be09c11a9894d54a67b7f1d7a4c130d06a8e76b0a8c22ffce74575",
"sha256:7b82f6eb19ba0cdc0690191a33ba626c79b30d95d781d20fa63ea10197de1a61",
"sha256:7ba143cce3fba6c2b115dc2aa88a6965d5208df3293a0c1d81f5234a1d88b728",
"sha256:80bf63fa9493c82d72788376c7342a27b8c4f07971be405cbb886c6b025bebc8",
"sha256:84389e0e8f4cf11fed55815a1eb7e4dc89513efedaf2a11e21ef1486a941b535",
"sha256:855d0357b162fe6053649eb20f05c3ebeaeb19bf2772d878d518f9928ec8e3b2",
"sha256:87a7988c1b741dd41a935aafc5927aa2ccd5672745f1f17994783c76b7912dd9",
"sha256:8a724c63cf0e2089756c88e1257fe7f6261bf38711375cd1b1a9dbd68cbe2384",
"sha256:8b1560aab51efe0ffb27eeafb7f1be3f138f8c7601198659de56ac36ae189f84",
"sha256:8c77926ba1e4015a3d6c30d788e116ea70fa773227957afde8bd8a61a58e13f1",
"sha256:9b1082f581660adf90b902b99e526cfd970ae9cd988d0ff3322300f60838fee4",
"sha256:9c14cf1b3945d86dcf0d8b404d1327185a03511caa1d3225eeb344b74a85b138",
"sha256:9f49ca00e49dac93663927c4c8c2758b97463bab65ceb932157cc4c114d39d21",
"sha256:a511dfeb956f44ceacd5e9e23cf325da240c347a406b31dd536aefa120d0fefb",
"sha256:a574db4c96e38ddb8a40f1dbf3c906949e9923420a480810d22b7b78f61e0596",
"sha256:a84f39e0efe7dcc8d2a34be976c2c5508bace6d703979730bb9021f6cce5dd0e",
"sha256:ac6b117abb4bd257d5f6b25cca060e74f3e2e038faa5c05d201335aa4657935d",
"sha256:b4fb3e647aca98c00eaf9f15de89a9287ebbcd873a8e3e3dcee6df91766a2734",
"sha256:bad2695ada7727f976c2aa3ad5a9ea68c486076deb8378e004b789e48f8d175b",
"sha256:bc5fa1a9b29c0f2ab2f706c5079b44f35002530ae39b609b6fcb80e5a70178e3",
"sha256:e34e85a2868a5853f3f66ef1b04700e3e2ca92f0af09dc4060e57e8a295b2871",
"sha256:e42e969cb4292a550b54f5436b6f52cc35fd5b6209859710c42892368308a544",
"sha256:fedc1f64714a32c01503f39ac3da2ed21dbceaa534107b3c9c617be63971d7b6"
],
"index": "pypi",
"version": "==2.7.5"
"version": "==2.7.6"
},
"pyasn1": {
"hashes": [
......@@ -588,10 +553,10 @@
},
"requests": {
"hashes": [
"sha256:99dcfdaaeb17caf6e526f32b6a7b780461512ab3f1d992187801694cba42770c",
"sha256:a84b8c9ab6239b578f22d1c21d51b696dcfe004032bb80ea832398d6909d7279"
"sha256:65b3a120e4329e33c9889db89c80976c5272f56ea92d3e74da8a463992e3ff54",
"sha256:ea881206e59f41dbd0bd445437d792e43906703fff75ca8ff43ccdb11f33f263"
],
"version": "==2.20.0"
"version": "==2.20.1"
},
"rjsmin": {
"hashes": [
......@@ -628,13 +593,6 @@
"index": "pypi",
"version": "==2.0.17.1"
},
"vine": {
"hashes": [
"sha256:52116d59bc45392af9fdd3b75ed98ae48a93e822cee21e5fda249105c59a7a72",
"sha256:6849544be74ec3638e84d90bc1cf2e1e9224cc10d96cd4383ec3f69e9bce077b"
],
"version": "==1.1.4"
},
"webencodings": {
"hashes": [
"sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78",
......
......@@ -7,11 +7,6 @@ services:
- /var/lib/postgresql/
environment: &postgresvars
POSTGRES_DB: thalia
redis:
image: redis:4.0.9
command: redis-server --appendonly yes
volumes:
- redis:/data
web:
image: registry.gitlab.com/thaliawww/concrexit
build: .
......@@ -28,19 +23,6 @@ services:
<<: *postgresvars
DJANGO_DEBUG: 'True'
DJANGO_POSTGRES_HOST: postgres
CELERY_BROKER_URL: "redis://redis:6379/0"
celery:
image: registry.gitlab.com/thaliawww/concrexit
build: .
user: nobody
entrypoint: /usr/local/bin/entrypoint_celery.sh
volumes:
- ./website:/usr/src/app/website/
- concrexit:/concrexit/
depends_on:
- redis
environment:
<<: *webvars
volumes:
concrexit:
......
......@@ -40,14 +40,6 @@ pushnotifications.models module
:undoc-members:
:show-inheritance:
pushnotifications.tasks module
------------------------------
.. automodule:: pushnotifications.tasks
:members:
:undoc-members:
:show-inheritance:
pushnotifications.urls module
-----------------------------
......
......@@ -26,14 +26,6 @@ thaliawebsite.admin module
:undoc-members:
:show-inheritance:
thaliawebsite.celery module
---------------------------
.. automodule:: thaliawebsite.celery
:members:
:undoc-members:
:show-inheritance:
thaliawebsite.context\_processors module
----------------------------------------
......
......@@ -33,14 +33,6 @@ utils.snippets module
:undoc-members:
:show-inheritance:
utils.tasks module
------------------
.. automodule:: utils.tasks
:members:
:undoc-members:
:show-inheritance:
utils.threading module
----------------------
......
#!/bin/bash
set -e
# Wait for Redis server to start
sleep 10
# Could do the following if redis-tools installed:
# X="`redis-cli -h \"$CELERY_BROKER_HOST\" ping`"
# echo ${X}
#
# while [ "${X}" != "PONG" ]; do
# >&2 echo "Redis is unavailable: Sleeping"
# echo "${X}"
# sleep 5
# done
# >&2 echo "Redis is up"
cd /usr/src/app/website/
>&2 echo "Starting celery worker"
celery worker --app thaliawebsite --statedb /celery/state.db
......@@ -70,12 +70,14 @@ class ScheduledMessageAdmin(TranslatedModelAdmin):
def get_fields(self, request, obj=None):
if obj and obj.sent:
return ('users', 'title_nl', 'title_en', 'body_nl', 'body_en',
'url', 'category', 'success', 'failure', 'time', 'task_id')
'url', 'category', 'success', 'failure', 'time',
'executed')
return ('users', 'title_nl', 'title_en', 'body_nl', 'body_en',
'url', 'category', 'time', 'task_id')
'url', 'category', 'time')
def get_readonly_fields(self, request, obj=None):
if obj and obj.sent:
return ('users', 'title_nl', 'title_en', 'body_nl', 'body_en',
'url', 'category', 'success', 'failure', 'time', 'task_id')
return 'task_id',
'url', 'category', 'success', 'failure', 'time',
'executed')
return 'executed',
import logging
from datetime import timedelta
from django.core.management.base import BaseCommand
from django.utils import timezone
from pushnotifications.models import ScheduledMessage
logger = logging.getLogger(__name__)
class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument(
'--dry-run',
action='store_true',
dest='dry-run',
default=False,
help='Dry run instead of sending notifications',
)
parser.add_argument(
'--interval',
dest='interval',
default=300,
help='Interval in seconds in which this task is executed',
)
def handle(self, *args, **options):
"""Send a scheduled push notifications"""
interval = int(options['interval'])
now = timezone.now()
logger.info('Start sending scheduled notifications')
before_time = timezone.now() + timedelta(seconds=interval/2)
messages = ScheduledMessage.objects.filter(
sent=False, time__lte=before_time)
for message in messages:
if (timezone.now() - now).seconds < interval:
logger.info(f'Sending push notification {message.pk}')
message.executed = timezone.now()
message.send(dry_run=bool(options['dry-run']))
# Generated by Django 2.1.3 on 2018-11-11 11:38
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('pushnotifications', '0012_add_descriptions_categories'),
]
operations = [
migrations.RemoveField(
model_name='scheduledmessage',
name='task_id',
),
migrations.AddField(
model_name='scheduledmessage',
name='executed',
field=models.DateTimeField(null=True),
),
]
......@@ -7,9 +7,7 @@ from django.utils.translation import override
from django.utils.translation import ugettext_lazy as _
from firebase_admin import messaging
from utils.tasks import revoke_task, schedule_task
from utils.translation import MultilingualField, ModelTranslateMeta
from .tasks import send_message
class Category(models.Model, metaclass=ModelTranslateMeta):
......@@ -166,7 +164,8 @@ class Message(models.Model, metaclass=ModelTranslateMeta):
for reg_id in reg_ids:
message.token = reg_id
try:
messaging.send(message)
messaging.send(message,dry_run=kwargs.get(
'dry_run', False))
success_total += 1
except messaging.ApiCallError as e:
failure_total += 1
......@@ -178,11 +177,10 @@ class Message(models.Model, metaclass=ModelTranslateMeta):
or e.code == 'invalid-registration-token'):
d.update(active=False)
if success_total > 0 or failure_total > 0:
self.sent = True
self.success = success_total
self.failure = failure_total
self.save()
self.sent = True
self.success = success_total
self.failure = failure_total
self.save()
return None
......@@ -198,30 +196,5 @@ class ScheduledMessage(Message, metaclass=ModelTranslateMeta):
objects = ScheduledMessageManager()
task_id = models.CharField(max_length=50, blank=True, null=True)
time = models.DateTimeField()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._time = self.time
def schedule(self):
"""Schedules a Celery task to send this message"""
return schedule_task(send_message, args=(self.pk,), eta=self.time)
def save(self, *args, **kwargs):
"""Custom save method which also schedules the task"""
if not (self._time == self.time):
if self.task_id:
# Revoke that task in case its time has changed
revoke_task(self.task_id)
super().save(*args, **kwargs)
self.task_id = self.schedule()
super().save(*args, **kwargs)
def delete(self, using=None, keep_parents=False):
if self.task_id:
revoke_task(self.task_id)
return super().delete(using, keep_parents)
executed = models.DateTimeField(null=True)
"""The celery tasks defined by the pushnotifications package"""
from celery import shared_task
from django.apps import apps
@shared_task
def send_message(message_id):
"""Send a scheduled push notification"""
print('Sending push notification {}'.format(message_id))
ScheduledMessage = apps.get_model('pushnotifications', 'ScheduledMessage')
try:
message = ScheduledMessage.objects.get(pk=message_id)
except ScheduledMessage.DoesNotExist:
print('Cannot find ScheduledMessage')
return
message.send()
......@@ -4,8 +4,3 @@ The main module for the Thalia website.
This module defines settings and the URI layout.
We also handle some site-wide API stuff here.
"""
# This will make sure the app is always imported when
# Django starts so that shared_task will use this app.
from .celery import app as celery_app
__all__ = ['celery_app']
"""Celery entry point"""
import os
from celery import Celery
# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'thaliawebsite.settings')
app = Celery('thaliawebsite') # pylint: disable=invalid-name
# Using a string here means the worker doesn't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
# should have a `CELERY_` prefix.
app.config_from_object('django.conf:settings', namespace='CELERY')
# Load task modules from all registered Django app configs.
app.autodiscover_tasks()
......@@ -7,7 +7,7 @@ Using environment variables you can control the loading of various
overrides.
"""
# flake8: noqa: ignore F403
import logging
from firebase_admin import initialize_app, credentials
# Load all default settings because we need to use settings.configure
......@@ -17,6 +17,8 @@ from django.conf.global_settings import * # pylint: disable=wildcard-import
# Load base settings
from .settings import * # pylint: disable=wildcard-import
logger = logging.getLogger(__name__)
# Attempt to load local overrides
try:
from .localsettings import * # pylint: disable=wildcard-import
......@@ -35,4 +37,4 @@ try:
initialize_app(
credential=credentials.Certificate(FIREBASE_CREDENTIALS))
except ValueError as e:
print('Firebase application failed to initialise')
logger.error('Firebase application failed to initialise')
......@@ -130,10 +130,6 @@ if os.environ.get('DJANGO_EMAIL_HOST'):
EMAIL_USE_SSL = os.environ.get('DJANGO_EMAIL_USE_SSL', False) == 'True'
EMAIL_TIMEOUT = 10
# Celery settings
CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL')
CELERY_ENABLED = True
LOGGING = {
'version': 1,
'disable_existing_loggers': True,
......
......@@ -56,7 +56,6 @@ INSTALLED_APPS = [
'rest_framework.authtoken',
'compressor',
'corsheaders',
'django_celery_results',
# Our apps
# Directly link to the app config when applicable as recommended
# by the docs: https://docs.djangoproject.com/en/2.0/ref/applications/
......@@ -244,14 +243,6 @@ COMPRESS_CSS_FILTERS = ['compressor.filters.css_default.CssAbsoluteFilter',
# Precompiler settings
STATIC_PRECOMPILER_LIST_FILES = True
# Celery settings
CELERY_BROKER_URL = 'redis://localhost:6379/0'
# Checkout caveats for the timeout config below:
# http://docs.celeryproject.org/en/latest/getting-started/brokers/redis.html#id1
CELERY_BROKER_TRANSPORT_OPTIONS = {'visibility_timeout': 15778800}
CELERY_RESULT_BACKEND = 'django-db'
CELERY_ENABLED = False
# Membership prices
MEMBERSHIP_PRICES = {
'year': 7.5,
......
......@@ -41,6 +41,3 @@ _ = [MIDDLEWARE.remove(x) for x in (
'django.middleware.csrf.CsrfViewMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
)]
# Celery not needed for testing
CELERY_ENABLED = False
from django.utils import timezone
from thaliawebsite import celery_app
from django.conf import settings
def schedule_task(task, args=(), eta=timezone.now()):
if settings.CELERY_ENABLED:
result = task.apply_async(args, eta=eta)
return result.id
return None
def revoke_task(task_id):