Remove Celery dependency

Dockerfile

@@ -56,10 +56,8 @@ RUN pip install --no-cache-dir \
 # Create entry points
 COPY resources/entrypoint.sh /usr/local/bin/entrypoint.sh
 COPY resources/entrypoint_production.sh /usr/local/bin/entrypoint_production.sh
-COPY resources/entrypoint_celery.sh /usr/local/bin/entrypoint_celery.sh
 RUN chmod +x /usr/local/bin/entrypoint.sh && \
-    chmod +x /usr/local/bin/entrypoint_production.sh && \
-    chmod +x /usr/local/bin/entrypoint_celery.sh
+    chmod +x /usr/local/bin/entrypoint_production.sh
 
 # copy app source
 COPY website /usr/src/app/website/

Pipfile

@@ -15,13 +15,11 @@ django-ical = "*"
 django-libsass = "*"
 django-cors-headers = "*"
 python-magic = "*"
-celery = "*"
 redis = "*"
 raven = "*"
 Django = ">=2.1,<2.2"
 Pillow = "*"
 django_compressor = "*"
-django_celery_results = "*"
 "psycopg2-binary" = "*"
 bcrypt = "*"
 "argon2_cffi" = "*"

Pipfile.lock

 {
     "_meta": {
         "hash": {
-            "sha256": "578ce12347a823b00a53ce5996d4778ba357b368eaf677a6b767ebdfa79d7f80"
+            "sha256": "24422ed4d4ab21978c31bc550d14ef87328366a5ebaf3903efd91ddbe34af434"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -14,13 +14,6 @@
         ]
     },
     "default": {
-        "amqp": {
-            "hashes": [
-                "sha256:073dd02fdd73041bffc913b767866015147b61f2a9bc104daef172fc1a0066eb",
-                "sha256:eed41946890cd43e8dee44a316b85cf6fee5a1a34bb4a562b660a358eb529e1b"
-            ],
-            "version": "==2.3.2"
-        },
         "argon2-cffi": {
             "hashes": [
                 "sha256:003f588de43a817af6ecc1c06103fa0801de63849db3cb0f37576bb2da29043d",
@@ -84,12 +77,6 @@
             "index": "pypi",
             "version": "==3.1.4"
         },
-        "billiard": {
-            "hashes": [
-                "sha256:ed65448da5877b5558f19d2f7f11f8355ea76b3e63e1c0a6059f47cfae5f1c84"
-            ],
-            "version": "==3.5.0.4"
-        },
         "bleach": {
             "hashes": [
                 "sha256:48d39675b80a75f6d1c3bdbffec791cf0bbbab665cf01e20da701c77de278718",
@@ -111,14 +98,6 @@
             ],
             "version": "==3.0.0"
         },
-        "celery": {
-            "hashes": [
-                "sha256:77dab4677e24dc654d42dfbdfed65fa760455b6bb563a0877ecc35f4cfcfc678",
-                "sha256:ad7a7411772b80a4d6c64f2f7f723200e39fb66cf614a7fdfab76d345acc7b13"
-            ],
-            "index": "pypi",
-            "version": "==4.2.1"
-        },
         "certifi": {
             "hashes": [
                 "sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c",
@@ -193,13 +172,6 @@
             "index": "pypi",
             "version": "==0.0.7"
         },
-        "django-celery-results": {
-            "hashes": [
-                "sha256:8bca2605eeff4418be7ce428a6958d64bee0f5bdf1f8e563fbc09a9e2f3d990f",
-                "sha256:dfa240fb535a1a2d01c9e605ad71629909318eae6b893c5009eafd7265fde10b"
-            ],
-            "version": "==1.0.1"
-        },
         "django-compressor": {
             "hashes": [
                 "sha256:7732676cfb9d58498dfb522b036f75f3f253f72ea1345ac036434fdc418c2e57",
@@ -278,17 +250,17 @@
         },
         "google-api-core": {
             "hashes": [
-                "sha256:35c0fdb7d0ea8e699d46611c31cdbbaef55c4b3905b394f16a8c41800be5de51",
-                "sha256:7cf597628cb9c5ceb24834b30a325dc271d3ba15d868d81c20aa80a77e13be65"
+                "sha256:383993eba1036c942f0d87497bac646b55ad8b4337d41527ce50e640768d769a",
+                "sha256:d32b2b9d31ef799b4ec5bcd35541ec2b3254e0d86a5606f2720d4ff2c62b4527"
             ],
-            "version": "==1.5.1"
+            "version": "==1.5.2"
         },
         "google-auth": {
             "hashes": [
-                "sha256:9ca363facbf2622d9ba828017536ccca2e0f58bd15e659b52f312172f8815530",
-                "sha256:a4cf9e803f2176b5de442763bd339b313d3f1ed3002e3e1eb6eec1d7c9bbc9b4"
+                "sha256:1cf96ac82fd37877e54c16264ebbb98b472de411ef87a4626cfc0b5b5df16d43",
+                "sha256:afd71b88f3ba873d86fe52aa2d352915ed579ca6b9fa3d3750c2e55656ae5aff"
             ],
-            "version": "==1.5.1"
+            "version": "==1.6.0"
         },
         "google-cloud-core": {
             "hashes": [
@@ -381,13 +353,6 @@
             ],
             "version": "==2.2.2"
         },
-        "kombu": {
-            "hashes": [
-                "sha256:86adec6c60f63124e2082ea8481bbe4ebe04fde8ebed32c177c7f0cd2c1c9082",
-                "sha256:b274db3a4eacc4789aeb24e1de3e460586db7c4fc8610f7adcc7a3a1709a60af"
-            ],
-            "version": "==4.2.1"
-        },
         "libsass": {
             "hashes": [
                 "sha256:2313f0e82de034eea59443c8f69420c60c55f7c07fd3b59ea7e7a108b36e9d86",
@@ -486,39 +451,39 @@
         },
         "psycopg2-binary": {
             "hashes": [
-                "sha256:04afb59bbbd2eab3148e6816beddc74348078b8c02a1113ea7f7822f5be4afe3",
-                "sha256:098b18f4d8857a8f9b206d1dc54db56c2255d5d26458917e7bcad61ebfe4338f",
-                "sha256:0bf855d4a7083e20ead961fda4923887094eaeace0ab2d76eb4aa300f4bbf5bd",
-                "sha256:197dda3ffd02057820be83fe4d84529ea70bf39a9a4daee1d20ffc74eb3d042e",
-                "sha256:278ef63afb4b3d842b4609f2c05ffbfb76795cf6a184deeb8707cd5ed3c981a5",
-                "sha256:3cbf8c4fc8f22f0817220891cf405831559f4d4c12c4f73913730a2ea6c47a47",
-                "sha256:4305aed922c4d9d6163ab3a41d80b5a1cfab54917467da8168552c42cad84d32",
-                "sha256:47ee296f704fb8b2a616dec691cdcfd5fa0f11943955e88faa98cbd1dc3b3e3d",
-                "sha256:4a0e38cb30457e70580903367161173d4a7d1381eb2f2cfe4e69b7806623f484",
-                "sha256:4d6c294c6638a71cafb82a37f182f24321f1163b08b5d5ca076e11fe838a3086",
-                "sha256:4f3233c366500730f839f92833194fd8f9a5c4529c8cd8040aa162c3740de8e5",
-                "sha256:5221f5a3f4ca2ddf0d58e8b8a32ca50948be9a43351fda797eb4e72d7a7aa34d",
-                "sha256:5c6ca0b507540a11eaf9e77dee4f07c131c2ec80ca0cffa146671bf690bc1c02",
-                "sha256:789bd89d71d704db2b3d5e67d6d518b158985d791d3b2dec5ab85457cfc9677b",
-                "sha256:7b94d29239efeaa6a967f3b5971bd0518d2a24edd1511edbf4a2c8b815220d07",
-                "sha256:89bc65ef3301c74cf32db25334421ea6adbe8f65601ea45dcaaf095abed910bb",
-                "sha256:89d6d3a549f405c20c9ae4dc94d7ed2de2fa77427a470674490a622070732e62",
-                "sha256:97521704ac7127d7d8ba22877da3c7bf4a40366587d238ec679ff38e33177498",
-                "sha256:a395b62d5f44ff6f633231abe568e2203b8fabf9797cd6386aa92497df912d9a",
-                "sha256:a6d32c37f714c3f34158f3fa659f3a8f2658d5f53c4297d45579b9677cc4d852",
-                "sha256:a89ee5c26f72f2d0d74b991ce49e42ddeb4ac0dc2d8c06a0f2770a1ab48f4fe0",
-                "sha256:b4c8b0ef3608e59317bfc501df84a61e48b5445d45f24d0391a24802de5f2d84",
-                "sha256:b5fcf07140219a1f71e18486b8dc28e2e1b76a441c19374805c617aa6d9a9d55",
-                "sha256:b86f527f00956ecebad6ab3bb30e3a75fedf1160a8716978dd8ce7adddedd86f",
-                "sha256:be4c4aa22ba22f70de36c98b06480e2f1697972d49eb20d525f400d204a6d272",
-                "sha256:c2ac7aa1a144d4e0e613ac7286dae85671e99fe7a1353954d4905629c36b811c",
-                "sha256:de26ef4787b5e778e8223913a3e50368b44e7480f83c76df1f51d23bd21cea16",
-                "sha256:e70ebcfc5372dc7b699c0110454fc4263967f30c55454397e5769eb72c0eb0ce",
-                "sha256:eadbd32b6bc48b67b0457fccc94c86f7ccc8178ab839f684eb285bb592dc143e",
-                "sha256:ecbc6dfff6db06b8b72ae8a2f25ff20fbdcb83cb543811a08f7cb555042aa729"
+                "sha256:0d3ca99a80d6018d3ee9723c4d7e5cbe04d20032cdb49469e7db5c8945951ac8",
+                "sha256:0f8a48ad5957bf420c3cc891417f1a41d363f33c00248d6c320d3910a2f7ab07",
+                "sha256:3187730658f8be85c0e761b49e7a3763507ebadd2c68be78a0153ef43a261c2d",
+                "sha256:35ecda69ea4ceb84e59dff47a6755b07069e45e1b18e50bccdd258da0bd42350",
+                "sha256:4bac49711e8b6b023600571259b0792795d00518a0f32df68ca2c4f8d7cda8ea",
+                "sha256:5f693cb5aa2ac94a62de3e45b2b0cba76c975daadeb4eaa26c64d302fcb7ca6c",
+                "sha256:6804e5d1cdba2e84dce38353e827ae3b2262f2c2f784675e18802ee6252cc289",
+                "sha256:6edd27d913be09c11a9894d54a67b7f1d7a4c130d06a8e76b0a8c22ffce74575",
+                "sha256:7b82f6eb19ba0cdc0690191a33ba626c79b30d95d781d20fa63ea10197de1a61",
+                "sha256:7ba143cce3fba6c2b115dc2aa88a6965d5208df3293a0c1d81f5234a1d88b728",
+                "sha256:80bf63fa9493c82d72788376c7342a27b8c4f07971be405cbb886c6b025bebc8",
+                "sha256:84389e0e8f4cf11fed55815a1eb7e4dc89513efedaf2a11e21ef1486a941b535",
+                "sha256:855d0357b162fe6053649eb20f05c3ebeaeb19bf2772d878d518f9928ec8e3b2",
+                "sha256:87a7988c1b741dd41a935aafc5927aa2ccd5672745f1f17994783c76b7912dd9",
+                "sha256:8a724c63cf0e2089756c88e1257fe7f6261bf38711375cd1b1a9dbd68cbe2384",
+                "sha256:8b1560aab51efe0ffb27eeafb7f1be3f138f8c7601198659de56ac36ae189f84",
+                "sha256:8c77926ba1e4015a3d6c30d788e116ea70fa773227957afde8bd8a61a58e13f1",
+                "sha256:9b1082f581660adf90b902b99e526cfd970ae9cd988d0ff3322300f60838fee4",
+                "sha256:9c14cf1b3945d86dcf0d8b404d1327185a03511caa1d3225eeb344b74a85b138",
+                "sha256:9f49ca00e49dac93663927c4c8c2758b97463bab65ceb932157cc4c114d39d21",
+                "sha256:a511dfeb956f44ceacd5e9e23cf325da240c347a406b31dd536aefa120d0fefb",
+                "sha256:a574db4c96e38ddb8a40f1dbf3c906949e9923420a480810d22b7b78f61e0596",
+                "sha256:a84f39e0efe7dcc8d2a34be976c2c5508bace6d703979730bb9021f6cce5dd0e",
+                "sha256:ac6b117abb4bd257d5f6b25cca060e74f3e2e038faa5c05d201335aa4657935d",
+                "sha256:b4fb3e647aca98c00eaf9f15de89a9287ebbcd873a8e3e3dcee6df91766a2734",
+                "sha256:bad2695ada7727f976c2aa3ad5a9ea68c486076deb8378e004b789e48f8d175b",
+                "sha256:bc5fa1a9b29c0f2ab2f706c5079b44f35002530ae39b609b6fcb80e5a70178e3",
+                "sha256:e34e85a2868a5853f3f66ef1b04700e3e2ca92f0af09dc4060e57e8a295b2871",
+                "sha256:e42e969cb4292a550b54f5436b6f52cc35fd5b6209859710c42892368308a544",
+                "sha256:fedc1f64714a32c01503f39ac3da2ed21dbceaa534107b3c9c617be63971d7b6"
             ],
             "index": "pypi",
-            "version": "==2.7.5"
+            "version": "==2.7.6"
         },
         "pyasn1": {
             "hashes": [
@@ -588,10 +553,10 @@
         },
         "requests": {
             "hashes": [
-                "sha256:99dcfdaaeb17caf6e526f32b6a7b780461512ab3f1d992187801694cba42770c",
-                "sha256:a84b8c9ab6239b578f22d1c21d51b696dcfe004032bb80ea832398d6909d7279"
+                "sha256:65b3a120e4329e33c9889db89c80976c5272f56ea92d3e74da8a463992e3ff54",
+                "sha256:ea881206e59f41dbd0bd445437d792e43906703fff75ca8ff43ccdb11f33f263"
             ],
-            "version": "==2.20.0"
+            "version": "==2.20.1"
         },
         "rjsmin": {
             "hashes": [
@@ -628,13 +593,6 @@
             "index": "pypi",
             "version": "==2.0.17.1"
         },
-        "vine": {
-            "hashes": [
-                "sha256:52116d59bc45392af9fdd3b75ed98ae48a93e822cee21e5fda249105c59a7a72",
-                "sha256:6849544be74ec3638e84d90bc1cf2e1e9224cc10d96cd4383ec3f69e9bce077b"
-            ],
-            "version": "==1.1.4"
-        },
         "webencodings": {
             "hashes": [
                 "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78",

docker-compose.yml

@@ -7,11 +7,6 @@ services:
             - /var/lib/postgresql/
         environment: &postgresvars
             POSTGRES_DB: thalia
-    redis:
-        image: redis:4.0.9
-        command: redis-server --appendonly yes
-        volumes:
-          - redis:/data
     web:
         image: registry.gitlab.com/thaliawww/concrexit
         build: .
@@ -28,19 +23,6 @@ services:
             <<: *postgresvars
             DJANGO_DEBUG: 'True'
             DJANGO_POSTGRES_HOST: postgres
-            CELERY_BROKER_URL: "redis://redis:6379/0"
-    celery:
-        image: registry.gitlab.com/thaliawww/concrexit
-        build: .
-        user: nobody
-        entrypoint: /usr/local/bin/entrypoint_celery.sh
-        volumes:
-            - ./website:/usr/src/app/website/
-            - concrexit:/concrexit/
-        depends_on:
-          - redis
-        environment:
-            <<: *webvars
 
 volumes:
     concrexit:

docs/pushnotifications.rst

@@ -40,14 +40,6 @@ pushnotifications.models module
     :undoc-members:
     :show-inheritance:
 
-pushnotifications.tasks module
-------------------------------
-
-.. automodule:: pushnotifications.tasks
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 pushnotifications.urls module
 -----------------------------
 

docs/thaliawebsite.rst

@@ -26,14 +26,6 @@ thaliawebsite.admin module
     :undoc-members:
     :show-inheritance:
 
-thaliawebsite.celery module
----------------------------
-
-.. automodule:: thaliawebsite.celery
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 thaliawebsite.context\_processors module
 ----------------------------------------
 

docs/utils.rst

@@ -33,14 +33,6 @@ utils.snippets module
     :undoc-members:
     :show-inheritance:
 
-utils.tasks module
-------------------
-
-.. automodule:: utils.tasks
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 utils.threading module
 ----------------------
 

resources/entrypoint_celery.sh

-#!/bin/bash
-
-set -e
-
-# Wait for Redis server to start
-sleep 10
-
-# Could do the following if redis-tools installed:
-# X="`redis-cli -h \"$CELERY_BROKER_HOST\" ping`"
-# echo ${X}
-#
-# while  [ "${X}" != "PONG" ]; do
-#     >&2 echo "Redis is unavailable: Sleeping"
-#     echo "${X}"
-#     sleep 5
-# done
-# >&2 echo "Redis is up"
-
-cd /usr/src/app/website/
->&2 echo "Starting celery worker"
-celery worker --app thaliawebsite --statedb /celery/state.db
-

website/pushnotifications/admin.py

@@ -70,12 +70,14 @@ class ScheduledMessageAdmin(TranslatedModelAdmin):
     def get_fields(self, request, obj=None):
         if obj and obj.sent:
             return ('users', 'title_nl', 'title_en', 'body_nl', 'body_en',
-                    'url', 'category', 'success', 'failure', 'time', 'task_id')
+                    'url', 'category', 'success', 'failure', 'time',
+                    'executed')
         return ('users', 'title_nl', 'title_en', 'body_nl', 'body_en',
-                'url', 'category', 'time', 'task_id')
+                'url', 'category', 'time')
 
     def get_readonly_fields(self, request, obj=None):
         if obj and obj.sent:
             return ('users', 'title_nl', 'title_en', 'body_nl', 'body_en',
-                    'url', 'category', 'success', 'failure', 'time', 'task_id')
-        return 'task_id',
+                    'url', 'category', 'success', 'failure', 'time',
+                    'executed')
+        return 'executed',

website/pushnotifications/management/commands/send_scheduled_messages.py

+import logging
+from datetime import timedelta
+
+from django.core.management.base import BaseCommand
+from django.utils import timezone
+
+from pushnotifications.models import ScheduledMessage
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--dry-run',
+            action='store_true',
+            dest='dry-run',
+            default=False,
+            help='Dry run instead of sending notifications',
+        )
+        parser.add_argument(
+            '--interval',
+            dest='interval',
+            default=300,
+            help='Interval in seconds in which this task is executed',
+        )
+
+    def handle(self, *args, **options):
+        """Send a scheduled push notifications"""
+        interval = int(options['interval'])
+        now = timezone.now()
+
+        logger.info('Start sending scheduled notifications')
+
+        before_time = timezone.now() + timedelta(seconds=interval/2)
+        messages = ScheduledMessage.objects.filter(
+            sent=False, time__lte=before_time)
+
+        for message in messages:
+            if (timezone.now() - now).seconds < interval:
+                logger.info(f'Sending push notification {message.pk}')
+                message.executed = timezone.now()
+                message.send(dry_run=bool(options['dry-run']))

website/pushnotifications/migrations/0013_auto_20181111_1238.py

+# Generated by Django 2.1.3 on 2018-11-11 11:38
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pushnotifications', '0012_add_descriptions_categories'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='scheduledmessage',
+            name='task_id',
+        ),
+        migrations.AddField(
+            model_name='scheduledmessage',
+            name='executed',
+            field=models.DateTimeField(null=True),
+        ),
+    ]

website/pushnotifications/models.py

@@ -7,9 +7,7 @@ from django.utils.translation import override
 from django.utils.translation import ugettext_lazy as _
 from firebase_admin import messaging
 
-from utils.tasks import revoke_task, schedule_task
 from utils.translation import MultilingualField, ModelTranslateMeta
-from .tasks import send_message
 
 
 class Category(models.Model, metaclass=ModelTranslateMeta):
@@ -166,7 +164,8 @@ class Message(models.Model, metaclass=ModelTranslateMeta):
                     for reg_id in reg_ids:
                         message.token = reg_id
                         try:
-                            messaging.send(message)
+                            messaging.send(message,dry_run=kwargs.get(
+                                'dry_run', False))
                             success_total += 1
                         except messaging.ApiCallError as e:
                             failure_total += 1
@@ -178,11 +177,10 @@ class Message(models.Model, metaclass=ModelTranslateMeta):
                                     or e.code == 'invalid-registration-token'):
                                 d.update(active=False)
 
-            if success_total > 0 or failure_total > 0:
-                self.sent = True
-                self.success = success_total
-                self.failure = failure_total
-                self.save()
+            self.sent = True
+            self.success = success_total
+            self.failure = failure_total
+            self.save()
         return None
 
 
@@ -198,30 +196,5 @@ class ScheduledMessage(Message, metaclass=ModelTranslateMeta):
 
     objects = ScheduledMessageManager()
 
-    task_id = models.CharField(max_length=50, blank=True, null=True)
     time = models.DateTimeField()
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self._time = self.time
-
-    def schedule(self):
-        """Schedules a Celery task to send this message"""
-        return schedule_task(send_message, args=(self.pk,), eta=self.time)
-
-    def save(self, *args, **kwargs):
-        """Custom save method which also schedules the task"""
-
-        if not (self._time == self.time):
-            if self.task_id:
-                # Revoke that task in case its time has changed
-                revoke_task(self.task_id)
-            super().save(*args, **kwargs)
-            self.task_id = self.schedule()
-
-        super().save(*args, **kwargs)
-
-    def delete(self, using=None, keep_parents=False):
-        if self.task_id:
-            revoke_task(self.task_id)
-        return super().delete(using, keep_parents)
+    executed = models.DateTimeField(null=True)

website/pushnotifications/tasks.py

-"""The celery tasks defined by the pushnotifications package"""
-from celery import shared_task
-from django.apps import apps
-
-
-@shared_task
-def send_message(message_id):
-    """Send a scheduled push notification"""
-
-    print('Sending push notification {}'.format(message_id))
-
-    ScheduledMessage = apps.get_model('pushnotifications', 'ScheduledMessage')
-    try:
-        message = ScheduledMessage.objects.get(pk=message_id)
-    except ScheduledMessage.DoesNotExist:
-        print('Cannot find ScheduledMessage')
-        return
-
-    message.send()

website/thaliawebsite/__init__.py

@@ -4,8 +4,3 @@ The main module for the Thalia website.
 This module defines settings and the URI layout.
 We also handle some site-wide API stuff here.