From 78af8b13aa03de98131a577f3c7fb5935ba0b1d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Se=CC=81bastiaan=20Versteeg?= Date: Mon, 5 Nov 2018 00:31:43 +0100 Subject: [PATCH 1/5] Replace pyFCM by Firebase Admin SDK --- Pipfile | 2 +- Pipfile.lock | 193 +++++++++++++++++-- website/pushnotifications/models.py | 75 ++++--- website/thaliawebsite/settings/__init__.py | 9 +- website/thaliawebsite/settings/production.py | 6 +- website/thaliawebsite/settings/settings.py | 6 +- 6 files changed, 222 insertions(+), 69 deletions(-) diff --git a/Pipfile b/Pipfile index 13be2e71..26a81a7c 100644 --- a/Pipfile +++ b/Pipfile @@ -15,7 +15,6 @@ django-ical = "*" django-libsass = "*" django-cors-headers = "*" python-magic = "*" -pyfcm = "*" celery = "*" redis = "*" raven = "*" @@ -28,6 +27,7 @@ bcrypt = "*" "argon2_cffi" = "*" uWSGI = "*" "django-bootstrap4" = "*" +firebase-admin = "*" [dev-packages] django-template-check = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 88d801dc..fcfede2d 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "351d600cf5450eb307c94e01e950568ba6b0283c36697cb59013d315868580b7" + "sha256": "578ce12347a823b00a53ce5996d4778ba357b368eaf677a6b767ebdfa79d7f80" }, "pipfile-spec": 6, "requires": {}, @@ -98,6 +98,19 @@ "index": "pypi", "version": "==3.0.2" }, + "cachecontrol": { + "hashes": [ + "sha256:cef77effdf51b43178f6a2d3b787e3734f98ade253fa3187f3bb7315aaa42ff7" + ], + "version": "==0.12.5" + }, + "cachetools": { + "hashes": [ + "sha256:0a258d82933a1dd18cb540aca4ac5d5690731e24d1239a08577b814998f49785", + "sha256:4621965b0d9d4c82a79a29edbad19946f5e7702df4afae7d1ed2df951559a8cc" + ], + "version": "==3.0.0" + }, "celery": { "hashes": [ "sha256:77dab4677e24dc654d42dfbdfed65fa760455b6bb563a0877ecc35f4cfcfc678", @@ -248,6 +261,13 @@ "index": "pypi", "version": "==3.9.0" }, + "firebase-admin": { + "hashes": [ + "sha256:4f74ecd2be9d4ba45272bf44e87f7d70bf73f01956b4d1b39ef56a02942c70b2" + ], + "index": "pypi", + "version": "==2.13.0" + }, "freezegun": { "hashes": [ "sha256:6cb82b276f83f2acce67f121dc2656f4df26c71e32238334eb071170b892a278", @@ -256,6 +276,91 @@ "index": "pypi", "version": "==0.3.11" }, + "google-api-core": { + "hashes": [ + "sha256:35c0fdb7d0ea8e699d46611c31cdbbaef55c4b3905b394f16a8c41800be5de51", + "sha256:7cf597628cb9c5ceb24834b30a325dc271d3ba15d868d81c20aa80a77e13be65" + ], + "version": "==1.5.1" + }, + "google-auth": { + "hashes": [ + "sha256:9ca363facbf2622d9ba828017536ccca2e0f58bd15e659b52f312172f8815530", + "sha256:a4cf9e803f2176b5de442763bd339b313d3f1ed3002e3e1eb6eec1d7c9bbc9b4" + ], + "version": "==1.5.1" + }, + "google-cloud-core": { + "hashes": [ + "sha256:0090df83dbc5cb2405fa90844366d13176d1c0b48181c1807ab15f53be403f73", + "sha256:89e8140a288acec20c5e56159461d3afa4073570c9758c05d4e6cb7f2f8cc440" + ], + "version": "==0.28.1" + }, + "google-cloud-firestore": { + "hashes": [ + "sha256:198ba208c7d2d189770e9ad5d7d00db4978cd11acbe0fe5e858266bcde113668", + "sha256:7f990572ace890867bbbc63c9d700c1d2635ba4c799e05f30b6fdca490021243" + ], + "version": "==0.30.0" + }, + "google-cloud-storage": { + "hashes": [ + "sha256:936c859c47f8e94fd0005e98235a10d5e75828d2c6c3a8caacae18344a572a0a", + "sha256:fc32b9be41a45016ba2387e3ad23e70ccba399d626ef596409316f7cee477956" + ], + "version": "==1.13.0" + }, + "google-resumable-media": { + "hashes": [ + "sha256:116de90b9cd483b17c53618ee6a5a20f33e741c648140c8cc9c2141e07616ff1", + "sha256:97de518f8166d442cc0b61fab308bcd319dbb970981e667ec8ded44f5ce49836" + ], + "version": "==0.3.1" + }, + "googleapis-common-protos": { + "hashes": [ + "sha256:0946967c4c29b1339bb211949e1e17dbe0ae9ff8265fafa7bf4cf2164ef5a3b1" + ], + "version": "==1.5.5" + }, + "grpcio": { + "hashes": [ + "sha256:0b09e82027f27cb540999404acf1be19cb50073d76ab257d7369aa3730bec3c0", + "sha256:0cc5f2d3ee21c642d8982f197c83053fd3a8cbcd6a60240d8c87c6c256b10d57", + "sha256:13b498b0415715a4214574c67ac6d0d7b565a861eb4490238a828fac17a51506", + "sha256:314c557efecec7f901cf394beb184b31414f906785e4811d2392859576d4d7b5", + "sha256:32d2859b68e185d05d6b5f5814121e786088f5e3483da0a7359f5d7fc0401ee3", + "sha256:3bf1b9d72a05a855762c36bd458d3750bedb5fd7b957a44443a62facf80afba4", + "sha256:41614ec2df4776a7d1b46183543d5c508bfc4972f092ec1ea83e98f808e5fa4d", + "sha256:4a7fab9f8ed8352d63585d221ee9c1fc58fb9b3d12535e777e36e855b0cab3db", + "sha256:4b4a2faa53e0f8d2b1479173dbce1523a7daaf2644fb835fb9fff04beb29ed8d", + "sha256:5526bf9f6615e22d0290aa83324f87fcc1fee51c3a9580ebeb2a52271c21a563", + "sha256:5bf2c9ec1d55c28ca1221f7b2d1914f20b2819c44579da89f447789baaba1386", + "sha256:62b24446d447ebe3a7002a6e3bd2c7372159e094868eb61ea2426327fe9f1992", + "sha256:63afda9d946fff727107ebbef25f6b45497f29486e462725dc9942391f3714a8", + "sha256:6dd039527b7333c947b9757ad40adf93b917f3734aed1da4fdeb28fd17ec63f0", + "sha256:6e719d17ca8fa06260a427cd1fab58abfd0672e8e625fcad81595bd125e0e367", + "sha256:76b3dbff4c775f5f8667c405b909ab2f80440c7579ad56f823476b011124a8a5", + "sha256:7be774ca3c8faa0e126d1e41e11fd82c9c114efb5437b36f651fe25add7f8c2e", + "sha256:7d74c3c6d8c7aadd505c8cef2b4b5324588bee645e6d20a6493940b24d394603", + "sha256:84afdfbf88c0ed2426a4f029fae3e677e8f1b2f3370feeae939d64670926c981", + "sha256:84d62107eb5bc9fe4e3682b038434c709ca7a2ae19e621e08ed7e8d908046cfb", + "sha256:8a1f4bee826b0edb123157f19843f46ca9ef29f12ed0b54eeffde5ff65101340", + "sha256:93edd492a1c6865e15db1ff7d98228b7351221bf815286a41834e10934c0cde0", + "sha256:9907fcb03a9fd327b114919dbb7a4577d5d5aeed2d6d000e6e6d002ad5cb959d", + "sha256:9dd008cd45a646b0e3761f0963c95b0dcd07d880d278a3c1ce23dd4ecb9cd174", + "sha256:a440935203be2581f68de7a4c5ca7ca22e948a21af70d7279ba9a2e32f73ae40", + "sha256:a9144b8a0f73be76aff348e4d558a5c3f43a8378a17c6327d56dbea8efda4aeb", + "sha256:b14629835e796f7905db2f7d10035958f995bae67bf9e652b13be156ed4a8457", + "sha256:b4fe851428b630bdf6f3a99c3761ce3d304b194162812fc1312bfe7bd138e620", + "sha256:c4318cea2d85f13811655e5d1c30fe97074aeb8105b16cc6da2d1d5d64a9f4f7", + "sha256:e46d3d702198d164474078140e008e8961e95dfb5a100f2890eb201c94c48c6e", + "sha256:e986100947cdafa2817701ffe616f2dc0221cc27eb301d654b9462b98ee62912", + "sha256:f94ae68c43b4bba0272e565882db2709d8827910ccc427f0a89d8cf070180f61" + ], + "version": "==1.16.0" + }, "icalendar": { "hashes": [ "sha256:07c2447a1d44cbb27c90b8c6a5c98e890cc1853c6223e2a52195cddec26c6356", @@ -301,6 +406,26 @@ ], "version": "==0.15.1" }, + "msgpack": { + "hashes": [ + "sha256:0b3b1773d2693c70598585a34ca2715873ba899565f0a7c9a1545baef7e7fbdc", + "sha256:0bae5d1538c5c6a75642f75a1781f3ac2275d744a92af1a453c150da3446138b", + "sha256:0ee8c8c85aa651be3aa0cd005b5931769eaa658c948ce79428766f1bd46ae2c3", + "sha256:1369f9edba9500c7a6489b70fdfac773e925342f4531f1e3d4c20ac3173b1ae0", + "sha256:22d9c929d1d539f37da3d1b0e16270fa9d46107beab8c0d4d2bddffffe895cee", + "sha256:2ff43e3247a1e11d544017bb26f580a68306cec7a6257d8818893c1fda665f42", + "sha256:31a98047355d34d047fcdb55b09cb19f633cf214c705a765bd745456c142130c", + "sha256:8767eb0032732c3a0da92cbec5ac186ef89a3258c6edca09161472ca0206c45f", + "sha256:8acc8910218555044e23826980b950e96685dc48124a290c86f6f41a296ea172", + "sha256:ab189a6365be1860a5ecf8159c248f12d33f79ea799ae9695fa6a29896dcf1d4", + "sha256:cfd6535feb0f1cf1c7cdb25773e965cc9f92928244a8c3ef6f8f8a8e1f7ae5c4", + "sha256:e274cd4480d8c76ec467a85a9c6635bbf2258f0649040560382ab58cabb44bcf", + "sha256:f86642d60dca13e93260187d56c2bef2487aa4d574a669e8ceefcf9f4c26fd00", + "sha256:f8a57cbda46a94ed0db55b73e6ab0c15e78b4ede8690fa491a0e55128d552bb0", + "sha256:fcea97a352416afcbccd7af9625159d80704a25c519c251c734527329bb20d0e" + ], + "version": "==0.5.6" + }, "pillow": { "hashes": [ "sha256:00203f406818c3f45d47bb8fe7e67d3feddb8dcbbd45a289a1de7dd789226360", @@ -338,6 +463,27 @@ "markers": "python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.3.*'", "version": "==5.3.0" }, + "protobuf": { + "hashes": [ + "sha256:10394a4d03af7060fa8a6e1cbf38cea44be1467053b0aea5bbfcb4b13c4b88c4", + "sha256:1489b376b0f364bcc6f89519718c057eb191d7ad6f1b395ffd93d1aa45587811", + "sha256:1931d8efce896981fe410c802fd66df14f9f429c32a72dd9cfeeac9815ec6444", + "sha256:196d3a80f93c537f27d2a19a4fafb826fb4c331b0b99110f985119391d170f96", + "sha256:46e34fdcc2b1f2620172d3a4885128705a4e658b9b62355ae5e98f9ea19f42c2", + "sha256:4b92e235a3afd42e7493b281c8b80c0c65cbef45de30f43d571d1ee40a1f77ef", + "sha256:574085a33ca0d2c67433e5f3e9a0965c487410d6cb3406c83bdaf549bfc2992e", + "sha256:59cd75ded98094d3cf2d79e84cdb38a46e33e7441b2826f3838dcc7c07f82995", + "sha256:5ee0522eed6680bb5bac5b6d738f7b0923b3cafce8c4b1a039a6107f0841d7ed", + "sha256:65917cfd5da9dfc993d5684643063318a2e875f798047911a9dd71ca066641c9", + "sha256:685bc4ec61a50f7360c9fd18e277b65db90105adbf9c79938bd315435e526b90", + "sha256:92e8418976e52201364a3174e40dc31f5fd8c147186d72380cbda54e0464ee19", + "sha256:9335f79d1940dfb9bcaf8ec881fb8ab47d7a2c721fb8b02949aab8bbf8b68625", + "sha256:a7ee3bb6de78185e5411487bef8bc1c59ebd97e47713cba3c460ef44e99b3db9", + "sha256:ceec283da2323e2431c49de58f80e1718986b79be59c266bb0509cbf90ca5b9e", + "sha256:fcfc907746ec22716f05ea96b7f41597dfe1a1c088f861efb8a0d4f4196a6f10" + ], + "version": "==3.6.1" + }, "psycopg2-binary": { "hashes": [ "sha256:04afb59bbbd2eab3148e6816beddc74348078b8c02a1113ea7f7822f5be4afe3", @@ -374,6 +520,20 @@ "index": "pypi", "version": "==2.7.5" }, + "pyasn1": { + "hashes": [ + "sha256:b9d3abc5031e61927c82d4d96c1cec1e55676c1a991623cfed28faea73cdd7ca", + "sha256:f58f2a3d12fd754aa123e9fa74fb7345333000a035f3921dbdaa08597aa53137" + ], + "version": "==0.4.4" + }, + "pyasn1-modules": { + "hashes": [ + "sha256:a0cf3e1842e7c60fde97cb22d275eb6f9524f5c5250489e292529de841417547", + "sha256:a38a8811ea784c0136abfdba73963876328f66172db21a05a82f9515909bfb4e" + ], + "version": "==0.2.2" + }, "pycparser": { "hashes": [ "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3" @@ -381,14 +541,6 @@ "markers": "python_version != '3.2.*' and python_version != '3.1.*' and python_version != '3.3.*' and python_version != '3.0.*' and python_version >= '2.7'", "version": "==2.19" }, - "pyfcm": { - "hashes": [ - "sha256:de3c7292e63186f30852c78cbbbc1bfa47b3996deb4e905be38e89a0692285ee", - "sha256:eca0d93481a441ecad865ac7d81b3b237a3e7a11cb872fd6961523391858af73" - ], - "index": "pypi", - "version": "==1.4.5" - }, "python-dateutil": { "hashes": [ "sha256:063df5763652e21de43de7d9e00ccf239f953a832941e37be541614732cdfc93", @@ -441,20 +593,19 @@ ], "version": "==2.20.0" }, - "requests-toolbelt": { - "hashes": [ - "sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237", - "sha256:f6a531936c6fa4c6cfce1b9c10d5c4f498d16528d2a54a22ca00011205a187b5" - ], - "markers": "python_version != '3.2.*' and python_version >= '2.6' and python_version != '3.1.*' and python_version != '3.3.*' and python_version != '3.0.*' and python_version < '4'", - "version": "==0.8.0" - }, "rjsmin": { "hashes": [ "sha256:dd9591aa73500b08b7db24367f8d32c6470021f39d5ab4e50c7c02e4401386f1" ], "version": "==1.0.12" }, + "rsa": { + "hashes": [ + "sha256:14ba45700ff1ec9eeb206a2ce76b32814958a98e372006c8fb76ba820211be66", + "sha256:1a836406405730121ae9823e19c6e806c62bbad73f890574fff50efa4122c487" + ], + "version": "==4.0" + }, "six": { "hashes": [ "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", @@ -464,11 +615,11 @@ }, "urllib3": { "hashes": [ - "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf", - "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5" + "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", + "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" ], - "markers": "python_version != '3.2.*' and python_version >= '2.6' and python_version != '3.1.*' and python_version != '3.3.*' and python_version != '3.0.*' and python_version < '4'", - "version": "==1.23" + "markers": "python_version >= '2.7' and python_version != '3.3.*' and python_version != '3.2.*' and python_version < '4' and python_version != '3.1.*' and python_version != '3.0.*'", + "version": "==1.24.1" }, "uwsgi": { "hashes": [ diff --git a/website/pushnotifications/models.py b/website/pushnotifications/models.py index df7cf174..d2af4434 100755 --- a/website/pushnotifications/models.py +++ b/website/pushnotifications/models.py @@ -1,9 +1,11 @@ """The models defined by the pushnotifications package""" +import datetime + from django.conf import settings from django.db import models from django.utils.translation import override from django.utils.translation import ugettext_lazy as _ -from pyfcm import FCMNotification +from firebase_admin import messaging from utils.tasks import revoke_task, schedule_task from utils.translation import MultilingualField, ModelTranslateMeta @@ -127,10 +129,9 @@ class Message(models.Model, metaclass=ModelTranslateMeta): def send(self, **kwargs): if self: - any_reg_ids = False success_total = 0 failure_total = 0 - result_list = [] + ttl = kwargs.get('ttl', 3600) for lang in settings.LANGUAGES: with override(lang[0]): @@ -142,52 +143,46 @@ class Message(models.Model, metaclass=ModelTranslateMeta): language=lang[0] ).values_list('registration_id', flat=True)) - if len(reg_ids) == 0: - continue - - any_reg_ids = True - - data = {} + data = kwargs.get('data', {}) if self.url is not None: data['url'] = self.url - result = FCMNotification( - api_key=settings.PUSH_NOTIFICATIONS_API_KEY - ).notify_multiple_devices( - registration_ids=reg_ids, - message_title=self.title, - message_body=str(self.body), - color='#E62272', - sound='default', - data_message=data, - **kwargs + message = messaging.Message( + notification=messaging.Notification( + title=self.title, + body=str(self.body), + ), + data=data, + android=messaging.AndroidConfig( + ttl=datetime.timedelta(seconds=ttl), + priority='normal', + notification=messaging.AndroidNotification( + color='#E62272', + sound='default', + ), + ), ) - results = result['results'] - for (index, item) in enumerate(results): - if 'error' in item: - reg_id = reg_ids[index] - - if (item['error'] == 'NotRegistered' - or item['error'] == 'InvalidRegistration'): - Device.objects.filter( - registration_id=reg_id).delete() - else: - Device.objects.filter( - registration_id=reg_id - ).update(active=False) - - success_total += result['success'] - failure_total += result['failure'] - result_list.append(result) - - if any_reg_ids: + for reg_id in reg_ids: + message.token = reg_id + try: + messaging.send(message) + success_total += 1 + except messaging.ApiCallError as e: + failure_total += 1 + d = Device.objects.filter(registration_id=reg_id) + if e.code == 'registration-token-not-registered': + d.delete() + elif (e.code == 'invalid-argument' + or e.code == 'invalid-recipient' + or e.code == 'invalid-registration-token'): + d.update(active=False) + + if success_total > 0 or failure_total > 0: self.sent = True self.success = success_total self.failure = failure_total self.save() - - return result_list return None diff --git a/website/thaliawebsite/settings/__init__.py b/website/thaliawebsite/settings/__init__.py index f8cbfdd2..36e2faec 100644 --- a/website/thaliawebsite/settings/__init__.py +++ b/website/thaliawebsite/settings/__init__.py @@ -8,13 +8,12 @@ overrides. """ # flake8: noqa: ignore F403 -import os +from firebase_admin import initialize_app, credentials # Load all default settings because we need to use settings.configure # for sphinx documentation generation. from django.conf.global_settings import * # pylint: disable=wildcard-import - # Load base settings from .settings import * # pylint: disable=wildcard-import @@ -31,3 +30,9 @@ if os.environ.get('DJANGO_PRODUCTION'): # pragma: nocover # Load testing settings if GITLAB_CI is set if os.environ.get('GITLAB_CI'): # pragma: nocover from .testing import * # pylint: disable=wildcard-import + +try: + initialize_app( + credential=credentials.Certificate(FIREBASE_CREDENTIALS)) +except ValueError as e: + print('Firebase application failed to initialise') diff --git a/website/thaliawebsite/settings/production.py b/website/thaliawebsite/settings/production.py index 53f91fd8..a2ecb924 100644 --- a/website/thaliawebsite/settings/production.py +++ b/website/thaliawebsite/settings/production.py @@ -8,7 +8,7 @@ This file is loaded by __init__.py if the environment variable See https://docs.djangoproject.com/en/dev/howto/deployment/checklist/ """ - +import json import os from . import settings @@ -76,12 +76,14 @@ PASSWORD_HASHERS = [ WIKI_API_KEY = os.environ.get('WIKI_API_KEY', 'changeme') MIGRATION_KEY = os.environ.get('MIGRATION_KEY') -PUSH_NOTIFICATIONS_API_KEY = os.environ.get('PUSH_NOTIFICATIONS_API_KEY', '') MAILINGLIST_API_SECRET = os.environ.get('MAILINGLIST_API_SECRET', '') MEMBERS_SENTRY_API_SECRET = os.environ.get('MEMBERS_SENTRY_API_SECRET', '') GOOGLE_MAPS_API_KEY = os.environ.get('GOOGLE_MAPS_API_KEY', '') GOOGLE_MAPS_API_SECRET = os.environ.get('GOOGLE_MAPS_API_SECRET', '') +FIREBASE_CREDENTIALS = os.environ.get('FIREBASE_CREDENTIALS', '{}') +if FIREBASE_CREDENTIALS.startswith('{'): + FIREBASE_CREDENTIALS = json.loads(FIREBASE_CREDENTIALS) if os.environ.get('DJANGO_SSLONLY'): SECURE_SSL_REDIRECT = True diff --git a/website/thaliawebsite/settings/settings.py b/website/thaliawebsite/settings/settings.py index be1c2a86..6d3c983e 100644 --- a/website/thaliawebsite/settings/settings.py +++ b/website/thaliawebsite/settings/settings.py @@ -265,6 +265,9 @@ THUMBNAIL_SIZES = { 'slide': '2000x430' } +# Placeholder Firebase config +FIREBASE_CREDENTIALS = {} + # Default FROM email DEFAULT_FROM_EMAIL = 'noreply@thalia.nu' SERVER_EMAIL = DEFAULT_FROM_EMAIL @@ -281,9 +284,6 @@ BOARD_NOTIFICATION_ADDRESS = 'info@thalia.nu' # Partners notification email PARTNER_EMAIL = "samenwerking@thalia.nu" -# Push notifications API key -PUSH_NOTIFICATIONS_API_KEY = '' - # Mailinglist API key MAILINGLIST_API_SECRET = '' -- GitLab From 188bbcc7519e5384e16367dacbeca29e4676f089 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Se=CC=81bastiaan=20Versteeg?= Date: Sun, 11 Nov 2018 13:11:07 +0100 Subject: [PATCH 2/5] Remove Celery dependency --- Dockerfile | 4 +- Pipfile | 2 - Pipfile.lock | 124 ++++++------------ docker-compose.yml | 18 --- docs/pushnotifications.rst | 8 -- docs/thaliawebsite.rst | 8 -- docs/utils.rst | 8 -- resources/entrypoint_celery.sh | 22 ---- website/pushnotifications/admin.py | 10 +- .../management/commands/__init__.py | 0 .../commands/send_scheduled_messages.py | 44 +++++++ .../migrations/0013_auto_20181111_1238.py | 22 ++++ website/pushnotifications/models.py | 41 +----- website/pushnotifications/tasks.py | 19 --- website/thaliawebsite/__init__.py | 5 - website/thaliawebsite/celery.py | 17 --- website/thaliawebsite/settings/__init__.py | 6 +- website/thaliawebsite/settings/production.py | 4 - website/thaliawebsite/settings/settings.py | 9 -- website/thaliawebsite/settings/testing.py | 3 - website/utils/tasks.py | 15 --- 21 files changed, 125 insertions(+), 264 deletions(-) delete mode 100644 resources/entrypoint_celery.sh create mode 100644 website/pushnotifications/management/commands/__init__.py create mode 100644 website/pushnotifications/management/commands/send_scheduled_messages.py create mode 100644 website/pushnotifications/migrations/0013_auto_20181111_1238.py delete mode 100644 website/pushnotifications/tasks.py delete mode 100644 website/thaliawebsite/celery.py delete mode 100644 website/utils/tasks.py diff --git a/Dockerfile b/Dockerfile index fbd6223f..a6ea4113 100644 --- a/Dockerfile +++ b/Dockerfile @@ -56,10 +56,8 @@ RUN pip install --no-cache-dir \ # Create entry points COPY resources/entrypoint.sh /usr/local/bin/entrypoint.sh COPY resources/entrypoint_production.sh /usr/local/bin/entrypoint_production.sh -COPY resources/entrypoint_celery.sh /usr/local/bin/entrypoint_celery.sh RUN chmod +x /usr/local/bin/entrypoint.sh && \ - chmod +x /usr/local/bin/entrypoint_production.sh && \ - chmod +x /usr/local/bin/entrypoint_celery.sh + chmod +x /usr/local/bin/entrypoint_production.sh # copy app source COPY website /usr/src/app/website/ diff --git a/Pipfile b/Pipfile index 26a81a7c..7652e6a3 100644 --- a/Pipfile +++ b/Pipfile @@ -15,13 +15,11 @@ django-ical = "*" django-libsass = "*" django-cors-headers = "*" python-magic = "*" -celery = "*" redis = "*" raven = "*" Django = ">=2.1,<2.2" Pillow = "*" django_compressor = "*" -django_celery_results = "*" "psycopg2-binary" = "*" bcrypt = "*" "argon2_cffi" = "*" diff --git a/Pipfile.lock b/Pipfile.lock index fcfede2d..ebc226bf 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "578ce12347a823b00a53ce5996d4778ba357b368eaf677a6b767ebdfa79d7f80" + "sha256": "24422ed4d4ab21978c31bc550d14ef87328366a5ebaf3903efd91ddbe34af434" }, "pipfile-spec": 6, "requires": {}, @@ -14,13 +14,6 @@ ] }, "default": { - "amqp": { - "hashes": [ - "sha256:073dd02fdd73041bffc913b767866015147b61f2a9bc104daef172fc1a0066eb", - "sha256:eed41946890cd43e8dee44a316b85cf6fee5a1a34bb4a562b660a358eb529e1b" - ], - "version": "==2.3.2" - }, "argon2-cffi": { "hashes": [ "sha256:003f588de43a817af6ecc1c06103fa0801de63849db3cb0f37576bb2da29043d", @@ -84,12 +77,6 @@ "index": "pypi", "version": "==3.1.4" }, - "billiard": { - "hashes": [ - "sha256:ed65448da5877b5558f19d2f7f11f8355ea76b3e63e1c0a6059f47cfae5f1c84" - ], - "version": "==3.5.0.4" - }, "bleach": { "hashes": [ "sha256:48d39675b80a75f6d1c3bdbffec791cf0bbbab665cf01e20da701c77de278718", @@ -111,14 +98,6 @@ ], "version": "==3.0.0" }, - "celery": { - "hashes": [ - "sha256:77dab4677e24dc654d42dfbdfed65fa760455b6bb563a0877ecc35f4cfcfc678", - "sha256:ad7a7411772b80a4d6c64f2f7f723200e39fb66cf614a7fdfab76d345acc7b13" - ], - "index": "pypi", - "version": "==4.2.1" - }, "certifi": { "hashes": [ "sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c", @@ -193,13 +172,6 @@ "index": "pypi", "version": "==0.0.7" }, - "django-celery-results": { - "hashes": [ - "sha256:8bca2605eeff4418be7ce428a6958d64bee0f5bdf1f8e563fbc09a9e2f3d990f", - "sha256:dfa240fb535a1a2d01c9e605ad71629909318eae6b893c5009eafd7265fde10b" - ], - "version": "==1.0.1" - }, "django-compressor": { "hashes": [ "sha256:7732676cfb9d58498dfb522b036f75f3f253f72ea1345ac036434fdc418c2e57", @@ -278,17 +250,17 @@ }, "google-api-core": { "hashes": [ - "sha256:35c0fdb7d0ea8e699d46611c31cdbbaef55c4b3905b394f16a8c41800be5de51", - "sha256:7cf597628cb9c5ceb24834b30a325dc271d3ba15d868d81c20aa80a77e13be65" + "sha256:383993eba1036c942f0d87497bac646b55ad8b4337d41527ce50e640768d769a", + "sha256:d32b2b9d31ef799b4ec5bcd35541ec2b3254e0d86a5606f2720d4ff2c62b4527" ], - "version": "==1.5.1" + "version": "==1.5.2" }, "google-auth": { "hashes": [ - "sha256:9ca363facbf2622d9ba828017536ccca2e0f58bd15e659b52f312172f8815530", - "sha256:a4cf9e803f2176b5de442763bd339b313d3f1ed3002e3e1eb6eec1d7c9bbc9b4" + "sha256:1cf96ac82fd37877e54c16264ebbb98b472de411ef87a4626cfc0b5b5df16d43", + "sha256:afd71b88f3ba873d86fe52aa2d352915ed579ca6b9fa3d3750c2e55656ae5aff" ], - "version": "==1.5.1" + "version": "==1.6.0" }, "google-cloud-core": { "hashes": [ @@ -381,13 +353,6 @@ ], "version": "==2.2.2" }, - "kombu": { - "hashes": [ - "sha256:86adec6c60f63124e2082ea8481bbe4ebe04fde8ebed32c177c7f0cd2c1c9082", - "sha256:b274db3a4eacc4789aeb24e1de3e460586db7c4fc8610f7adcc7a3a1709a60af" - ], - "version": "==4.2.1" - }, "libsass": { "hashes": [ "sha256:2313f0e82de034eea59443c8f69420c60c55f7c07fd3b59ea7e7a108b36e9d86", @@ -486,39 +451,39 @@ }, "psycopg2-binary": { "hashes": [ - "sha256:04afb59bbbd2eab3148e6816beddc74348078b8c02a1113ea7f7822f5be4afe3", - "sha256:098b18f4d8857a8f9b206d1dc54db56c2255d5d26458917e7bcad61ebfe4338f", - "sha256:0bf855d4a7083e20ead961fda4923887094eaeace0ab2d76eb4aa300f4bbf5bd", - "sha256:197dda3ffd02057820be83fe4d84529ea70bf39a9a4daee1d20ffc74eb3d042e", - "sha256:278ef63afb4b3d842b4609f2c05ffbfb76795cf6a184deeb8707cd5ed3c981a5", - "sha256:3cbf8c4fc8f22f0817220891cf405831559f4d4c12c4f73913730a2ea6c47a47", - "sha256:4305aed922c4d9d6163ab3a41d80b5a1cfab54917467da8168552c42cad84d32", - "sha256:47ee296f704fb8b2a616dec691cdcfd5fa0f11943955e88faa98cbd1dc3b3e3d", - "sha256:4a0e38cb30457e70580903367161173d4a7d1381eb2f2cfe4e69b7806623f484", - "sha256:4d6c294c6638a71cafb82a37f182f24321f1163b08b5d5ca076e11fe838a3086", - "sha256:4f3233c366500730f839f92833194fd8f9a5c4529c8cd8040aa162c3740de8e5", - "sha256:5221f5a3f4ca2ddf0d58e8b8a32ca50948be9a43351fda797eb4e72d7a7aa34d", - "sha256:5c6ca0b507540a11eaf9e77dee4f07c131c2ec80ca0cffa146671bf690bc1c02", - "sha256:789bd89d71d704db2b3d5e67d6d518b158985d791d3b2dec5ab85457cfc9677b", - "sha256:7b94d29239efeaa6a967f3b5971bd0518d2a24edd1511edbf4a2c8b815220d07", - "sha256:89bc65ef3301c74cf32db25334421ea6adbe8f65601ea45dcaaf095abed910bb", - "sha256:89d6d3a549f405c20c9ae4dc94d7ed2de2fa77427a470674490a622070732e62", - "sha256:97521704ac7127d7d8ba22877da3c7bf4a40366587d238ec679ff38e33177498", - "sha256:a395b62d5f44ff6f633231abe568e2203b8fabf9797cd6386aa92497df912d9a", - "sha256:a6d32c37f714c3f34158f3fa659f3a8f2658d5f53c4297d45579b9677cc4d852", - "sha256:a89ee5c26f72f2d0d74b991ce49e42ddeb4ac0dc2d8c06a0f2770a1ab48f4fe0", - "sha256:b4c8b0ef3608e59317bfc501df84a61e48b5445d45f24d0391a24802de5f2d84", - "sha256:b5fcf07140219a1f71e18486b8dc28e2e1b76a441c19374805c617aa6d9a9d55", - "sha256:b86f527f00956ecebad6ab3bb30e3a75fedf1160a8716978dd8ce7adddedd86f", - "sha256:be4c4aa22ba22f70de36c98b06480e2f1697972d49eb20d525f400d204a6d272", - "sha256:c2ac7aa1a144d4e0e613ac7286dae85671e99fe7a1353954d4905629c36b811c", - "sha256:de26ef4787b5e778e8223913a3e50368b44e7480f83c76df1f51d23bd21cea16", - "sha256:e70ebcfc5372dc7b699c0110454fc4263967f30c55454397e5769eb72c0eb0ce", - "sha256:eadbd32b6bc48b67b0457fccc94c86f7ccc8178ab839f684eb285bb592dc143e", - "sha256:ecbc6dfff6db06b8b72ae8a2f25ff20fbdcb83cb543811a08f7cb555042aa729" + "sha256:0d3ca99a80d6018d3ee9723c4d7e5cbe04d20032cdb49469e7db5c8945951ac8", + "sha256:0f8a48ad5957bf420c3cc891417f1a41d363f33c00248d6c320d3910a2f7ab07", + "sha256:3187730658f8be85c0e761b49e7a3763507ebadd2c68be78a0153ef43a261c2d", + "sha256:35ecda69ea4ceb84e59dff47a6755b07069e45e1b18e50bccdd258da0bd42350", + "sha256:4bac49711e8b6b023600571259b0792795d00518a0f32df68ca2c4f8d7cda8ea", + "sha256:5f693cb5aa2ac94a62de3e45b2b0cba76c975daadeb4eaa26c64d302fcb7ca6c", + "sha256:6804e5d1cdba2e84dce38353e827ae3b2262f2c2f784675e18802ee6252cc289", + "sha256:6edd27d913be09c11a9894d54a67b7f1d7a4c130d06a8e76b0a8c22ffce74575", + "sha256:7b82f6eb19ba0cdc0690191a33ba626c79b30d95d781d20fa63ea10197de1a61", + "sha256:7ba143cce3fba6c2b115dc2aa88a6965d5208df3293a0c1d81f5234a1d88b728", + "sha256:80bf63fa9493c82d72788376c7342a27b8c4f07971be405cbb886c6b025bebc8", + "sha256:84389e0e8f4cf11fed55815a1eb7e4dc89513efedaf2a11e21ef1486a941b535", + "sha256:855d0357b162fe6053649eb20f05c3ebeaeb19bf2772d878d518f9928ec8e3b2", + "sha256:87a7988c1b741dd41a935aafc5927aa2ccd5672745f1f17994783c76b7912dd9", + "sha256:8a724c63cf0e2089756c88e1257fe7f6261bf38711375cd1b1a9dbd68cbe2384", + "sha256:8b1560aab51efe0ffb27eeafb7f1be3f138f8c7601198659de56ac36ae189f84", + "sha256:8c77926ba1e4015a3d6c30d788e116ea70fa773227957afde8bd8a61a58e13f1", + "sha256:9b1082f581660adf90b902b99e526cfd970ae9cd988d0ff3322300f60838fee4", + "sha256:9c14cf1b3945d86dcf0d8b404d1327185a03511caa1d3225eeb344b74a85b138", + "sha256:9f49ca00e49dac93663927c4c8c2758b97463bab65ceb932157cc4c114d39d21", + "sha256:a511dfeb956f44ceacd5e9e23cf325da240c347a406b31dd536aefa120d0fefb", + "sha256:a574db4c96e38ddb8a40f1dbf3c906949e9923420a480810d22b7b78f61e0596", + "sha256:a84f39e0efe7dcc8d2a34be976c2c5508bace6d703979730bb9021f6cce5dd0e", + "sha256:ac6b117abb4bd257d5f6b25cca060e74f3e2e038faa5c05d201335aa4657935d", + "sha256:b4fb3e647aca98c00eaf9f15de89a9287ebbcd873a8e3e3dcee6df91766a2734", + "sha256:bad2695ada7727f976c2aa3ad5a9ea68c486076deb8378e004b789e48f8d175b", + "sha256:bc5fa1a9b29c0f2ab2f706c5079b44f35002530ae39b609b6fcb80e5a70178e3", + "sha256:e34e85a2868a5853f3f66ef1b04700e3e2ca92f0af09dc4060e57e8a295b2871", + "sha256:e42e969cb4292a550b54f5436b6f52cc35fd5b6209859710c42892368308a544", + "sha256:fedc1f64714a32c01503f39ac3da2ed21dbceaa534107b3c9c617be63971d7b6" ], "index": "pypi", - "version": "==2.7.5" + "version": "==2.7.6" }, "pyasn1": { "hashes": [ @@ -588,10 +553,10 @@ }, "requests": { "hashes": [ - "sha256:99dcfdaaeb17caf6e526f32b6a7b780461512ab3f1d992187801694cba42770c", - "sha256:a84b8c9ab6239b578f22d1c21d51b696dcfe004032bb80ea832398d6909d7279" + "sha256:65b3a120e4329e33c9889db89c80976c5272f56ea92d3e74da8a463992e3ff54", + "sha256:ea881206e59f41dbd0bd445437d792e43906703fff75ca8ff43ccdb11f33f263" ], - "version": "==2.20.0" + "version": "==2.20.1" }, "rjsmin": { "hashes": [ @@ -628,13 +593,6 @@ "index": "pypi", "version": "==2.0.17.1" }, - "vine": { - "hashes": [ - "sha256:52116d59bc45392af9fdd3b75ed98ae48a93e822cee21e5fda249105c59a7a72", - "sha256:6849544be74ec3638e84d90bc1cf2e1e9224cc10d96cd4383ec3f69e9bce077b" - ], - "version": "==1.1.4" - }, "webencodings": { "hashes": [ "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", diff --git a/docker-compose.yml b/docker-compose.yml index 07636de4..adb96849 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,11 +7,6 @@ services: - /var/lib/postgresql/ environment: &postgresvars POSTGRES_DB: thalia - redis: - image: redis:4.0.9 - command: redis-server --appendonly yes - volumes: - - redis:/data web: image: registry.gitlab.com/thaliawww/concrexit build: . @@ -28,19 +23,6 @@ services: <<: *postgresvars DJANGO_DEBUG: 'True' DJANGO_POSTGRES_HOST: postgres - CELERY_BROKER_URL: "redis://redis:6379/0" - celery: - image: registry.gitlab.com/thaliawww/concrexit - build: . - user: nobody - entrypoint: /usr/local/bin/entrypoint_celery.sh - volumes: - - ./website:/usr/src/app/website/ - - concrexit:/concrexit/ - depends_on: - - redis - environment: - <<: *webvars volumes: concrexit: diff --git a/docs/pushnotifications.rst b/docs/pushnotifications.rst index 291615b5..286570f2 100644 --- a/docs/pushnotifications.rst +++ b/docs/pushnotifications.rst @@ -40,14 +40,6 @@ pushnotifications.models module :undoc-members: :show-inheritance: -pushnotifications.tasks module ------------------------------- - -.. automodule:: pushnotifications.tasks - :members: - :undoc-members: - :show-inheritance: - pushnotifications.urls module ----------------------------- diff --git a/docs/thaliawebsite.rst b/docs/thaliawebsite.rst index c604f846..43c9271a 100644 --- a/docs/thaliawebsite.rst +++ b/docs/thaliawebsite.rst @@ -26,14 +26,6 @@ thaliawebsite.admin module :undoc-members: :show-inheritance: -thaliawebsite.celery module ---------------------------- - -.. automodule:: thaliawebsite.celery - :members: - :undoc-members: - :show-inheritance: - thaliawebsite.context\_processors module ---------------------------------------- diff --git a/docs/utils.rst b/docs/utils.rst index 14ff7198..7a7edc3b 100644 --- a/docs/utils.rst +++ b/docs/utils.rst @@ -33,14 +33,6 @@ utils.snippets module :undoc-members: :show-inheritance: -utils.tasks module ------------------- - -.. automodule:: utils.tasks - :members: - :undoc-members: - :show-inheritance: - utils.threading module ---------------------- diff --git a/resources/entrypoint_celery.sh b/resources/entrypoint_celery.sh deleted file mode 100644 index 2c64a6ab..00000000 --- a/resources/entrypoint_celery.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -set -e - -# Wait for Redis server to start -sleep 10 - -# Could do the following if redis-tools installed: -# X="`redis-cli -h \"$CELERY_BROKER_HOST\" ping`" -# echo ${X} -# -# while [ "${X}" != "PONG" ]; do -# >&2 echo "Redis is unavailable: Sleeping" -# echo "${X}" -# sleep 5 -# done -# >&2 echo "Redis is up" - -cd /usr/src/app/website/ ->&2 echo "Starting celery worker" -celery worker --app thaliawebsite --statedb /celery/state.db - diff --git a/website/pushnotifications/admin.py b/website/pushnotifications/admin.py index fce78928..56767cf9 100755 --- a/website/pushnotifications/admin.py +++ b/website/pushnotifications/admin.py @@ -70,12 +70,14 @@ class ScheduledMessageAdmin(TranslatedModelAdmin): def get_fields(self, request, obj=None): if obj and obj.sent: return ('users', 'title_nl', 'title_en', 'body_nl', 'body_en', - 'url', 'category', 'success', 'failure', 'time', 'task_id') + 'url', 'category', 'success', 'failure', 'time', + 'executed') return ('users', 'title_nl', 'title_en', 'body_nl', 'body_en', - 'url', 'category', 'time', 'task_id') + 'url', 'category', 'time') def get_readonly_fields(self, request, obj=None): if obj and obj.sent: return ('users', 'title_nl', 'title_en', 'body_nl', 'body_en', - 'url', 'category', 'success', 'failure', 'time', 'task_id') - return 'task_id', + 'url', 'category', 'success', 'failure', 'time', + 'executed') + return 'executed', diff --git a/website/pushnotifications/management/commands/__init__.py b/website/pushnotifications/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/website/pushnotifications/management/commands/send_scheduled_messages.py b/website/pushnotifications/management/commands/send_scheduled_messages.py new file mode 100644 index 00000000..fa5b6588 --- /dev/null +++ b/website/pushnotifications/management/commands/send_scheduled_messages.py @@ -0,0 +1,44 @@ +import logging +from datetime import timedelta + +from django.core.management.base import BaseCommand +from django.utils import timezone + +from pushnotifications.models import ScheduledMessage + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + + def add_arguments(self, parser): + parser.add_argument( + '--dry-run', + action='store_true', + dest='dry-run', + default=False, + help='Dry run instead of sending notifications', + ) + parser.add_argument( + '--interval', + dest='interval', + default=300, + help='Interval in seconds in which this task is executed', + ) + + def handle(self, *args, **options): + """Send a scheduled push notifications""" + interval = int(options['interval']) + now = timezone.now() + + logger.info('Start sending scheduled notifications') + + before_time = timezone.now() + timedelta(seconds=interval/2) + messages = ScheduledMessage.objects.filter( + sent=False, time__lte=before_time) + + for message in messages: + if (timezone.now() - now).seconds < interval: + logger.info(f'Sending push notification {message.pk}') + message.executed = timezone.now() + message.send(dry_run=bool(options['dry-run'])) diff --git a/website/pushnotifications/migrations/0013_auto_20181111_1238.py b/website/pushnotifications/migrations/0013_auto_20181111_1238.py new file mode 100644 index 00000000..a9fa1d74 --- /dev/null +++ b/website/pushnotifications/migrations/0013_auto_20181111_1238.py @@ -0,0 +1,22 @@ +# Generated by Django 2.1.3 on 2018-11-11 11:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('pushnotifications', '0012_add_descriptions_categories'), + ] + + operations = [ + migrations.RemoveField( + model_name='scheduledmessage', + name='task_id', + ), + migrations.AddField( + model_name='scheduledmessage', + name='executed', + field=models.DateTimeField(null=True), + ), + ] diff --git a/website/pushnotifications/models.py b/website/pushnotifications/models.py index d2af4434..f0fc67e6 100755 --- a/website/pushnotifications/models.py +++ b/website/pushnotifications/models.py @@ -7,9 +7,7 @@ from django.utils.translation import override from django.utils.translation import ugettext_lazy as _ from firebase_admin import messaging -from utils.tasks import revoke_task, schedule_task from utils.translation import MultilingualField, ModelTranslateMeta -from .tasks import send_message class Category(models.Model, metaclass=ModelTranslateMeta): @@ -166,7 +164,8 @@ class Message(models.Model, metaclass=ModelTranslateMeta): for reg_id in reg_ids: message.token = reg_id try: - messaging.send(message) + messaging.send(message,dry_run=kwargs.get( + 'dry_run', False)) success_total += 1 except messaging.ApiCallError as e: failure_total += 1 @@ -178,11 +177,10 @@ class Message(models.Model, metaclass=ModelTranslateMeta): or e.code == 'invalid-registration-token'): d.update(active=False) - if success_total > 0 or failure_total > 0: - self.sent = True - self.success = success_total - self.failure = failure_total - self.save() + self.sent = True + self.success = success_total + self.failure = failure_total + self.save() return None @@ -198,30 +196,5 @@ class ScheduledMessage(Message, metaclass=ModelTranslateMeta): objects = ScheduledMessageManager() - task_id = models.CharField(max_length=50, blank=True, null=True) time = models.DateTimeField() - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._time = self.time - - def schedule(self): - """Schedules a Celery task to send this message""" - return schedule_task(send_message, args=(self.pk,), eta=self.time) - - def save(self, *args, **kwargs): - """Custom save method which also schedules the task""" - - if not (self._time == self.time): - if self.task_id: - # Revoke that task in case its time has changed - revoke_task(self.task_id) - super().save(*args, **kwargs) - self.task_id = self.schedule() - - super().save(*args, **kwargs) - - def delete(self, using=None, keep_parents=False): - if self.task_id: - revoke_task(self.task_id) - return super().delete(using, keep_parents) + executed = models.DateTimeField(null=True) diff --git a/website/pushnotifications/tasks.py b/website/pushnotifications/tasks.py deleted file mode 100644 index 48d95702..00000000 --- a/website/pushnotifications/tasks.py +++ /dev/null @@ -1,19 +0,0 @@ -"""The celery tasks defined by the pushnotifications package""" -from celery import shared_task -from django.apps import apps - - -@shared_task -def send_message(message_id): - """Send a scheduled push notification""" - - print('Sending push notification {}'.format(message_id)) - - ScheduledMessage = apps.get_model('pushnotifications', 'ScheduledMessage') - try: - message = ScheduledMessage.objects.get(pk=message_id) - except ScheduledMessage.DoesNotExist: - print('Cannot find ScheduledMessage') - return - - message.send() diff --git a/website/thaliawebsite/__init__.py b/website/thaliawebsite/__init__.py index 99ff4bab..f13b0ba6 100644 --- a/website/thaliawebsite/__init__.py +++ b/website/thaliawebsite/__init__.py @@ -4,8 +4,3 @@ The main module for the Thalia website. This module defines settings and the URI layout. We also handle some site-wide API stuff here. """ -# This will make sure the app is always imported when -# Django starts so that shared_task will use this app. -from .celery import app as celery_app - -__all__ = ['celery_app'] diff --git a/website/thaliawebsite/celery.py b/website/thaliawebsite/celery.py deleted file mode 100644 index 9fba2906..00000000 --- a/website/thaliawebsite/celery.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Celery entry point""" -import os -from celery import Celery - -# set the default Django settings module for the 'celery' program. -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'thaliawebsite.settings') - -app = Celery('thaliawebsite') # pylint: disable=invalid-name - -# Using a string here means the worker doesn't have to serialize -# the configuration object to child processes. -# - namespace='CELERY' means all celery-related configuration keys -# should have a `CELERY_` prefix. -app.config_from_object('django.conf:settings', namespace='CELERY') - -# Load task modules from all registered Django app configs. -app.autodiscover_tasks() diff --git a/website/thaliawebsite/settings/__init__.py b/website/thaliawebsite/settings/__init__.py index 36e2faec..05008b64 100644 --- a/website/thaliawebsite/settings/__init__.py +++ b/website/thaliawebsite/settings/__init__.py @@ -7,7 +7,7 @@ Using environment variables you can control the loading of various overrides. """ # flake8: noqa: ignore F403 - +import logging from firebase_admin import initialize_app, credentials # Load all default settings because we need to use settings.configure @@ -17,6 +17,8 @@ from django.conf.global_settings import * # pylint: disable=wildcard-import # Load base settings from .settings import * # pylint: disable=wildcard-import +logger = logging.getLogger(__name__) + # Attempt to load local overrides try: from .localsettings import * # pylint: disable=wildcard-import @@ -35,4 +37,4 @@ try: initialize_app( credential=credentials.Certificate(FIREBASE_CREDENTIALS)) except ValueError as e: - print('Firebase application failed to initialise') + logger.error('Firebase application failed to initialise') diff --git a/website/thaliawebsite/settings/production.py b/website/thaliawebsite/settings/production.py index a2ecb924..87fa624b 100644 --- a/website/thaliawebsite/settings/production.py +++ b/website/thaliawebsite/settings/production.py @@ -130,10 +130,6 @@ if os.environ.get('DJANGO_EMAIL_HOST'): EMAIL_USE_SSL = os.environ.get('DJANGO_EMAIL_USE_SSL', False) == 'True' EMAIL_TIMEOUT = 10 -# Celery settings -CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL') -CELERY_ENABLED = True - LOGGING = { 'version': 1, 'disable_existing_loggers': True, diff --git a/website/thaliawebsite/settings/settings.py b/website/thaliawebsite/settings/settings.py index 6d3c983e..f0037e5b 100644 --- a/website/thaliawebsite/settings/settings.py +++ b/website/thaliawebsite/settings/settings.py @@ -56,7 +56,6 @@ INSTALLED_APPS = [ 'rest_framework.authtoken', 'compressor', 'corsheaders', - 'django_celery_results', # Our apps # Directly link to the app config when applicable as recommended # by the docs: https://docs.djangoproject.com/en/2.0/ref/applications/ @@ -244,14 +243,6 @@ COMPRESS_CSS_FILTERS = ['compressor.filters.css_default.CssAbsoluteFilter', # Precompiler settings STATIC_PRECOMPILER_LIST_FILES = True -# Celery settings -CELERY_BROKER_URL = 'redis://localhost:6379/0' -# Checkout caveats for the timeout config below: -# http://docs.celeryproject.org/en/latest/getting-started/brokers/redis.html#id1 -CELERY_BROKER_TRANSPORT_OPTIONS = {'visibility_timeout': 15778800} -CELERY_RESULT_BACKEND = 'django-db' -CELERY_ENABLED = False - # Membership prices MEMBERSHIP_PRICES = { 'year': 7.5, diff --git a/website/thaliawebsite/settings/testing.py b/website/thaliawebsite/settings/testing.py index db508acd..0b939841 100644 --- a/website/thaliawebsite/settings/testing.py +++ b/website/thaliawebsite/settings/testing.py @@ -41,6 +41,3 @@ _ = [MIDDLEWARE.remove(x) for x in ( 'django.middleware.csrf.CsrfViewMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', )] - -# Celery not needed for testing -CELERY_ENABLED = False diff --git a/website/utils/tasks.py b/website/utils/tasks.py deleted file mode 100644 index aa270415..00000000 --- a/website/utils/tasks.py +++ /dev/null @@ -1,15 +0,0 @@ -from django.utils import timezone -from thaliawebsite import celery_app -from django.conf import settings - - -def schedule_task(task, args=(), eta=timezone.now()): - if settings.CELERY_ENABLED: - result = task.apply_async(args, eta=eta) - return result.id - return None - - -def revoke_task(task_id): - if settings.CELERY_ENABLED: - celery_app.control.revoke(task_id) -- GitLab From 8c95f5f1f5ba220aa98c3ddec624574d31f935cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Se=CC=81bastiaan=20Versteeg?= Date: Sun, 11 Nov 2018 13:11:21 +0100 Subject: [PATCH 3/5] Add base url to notification url --- website/events/models.py | 13 +++++++------ website/members/emails.py | 6 +++--- website/registrations/emails.py | 2 +- website/thaliawebsite/settings/settings.py | 1 + 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/website/events/models.py b/website/events/models.py index 63e9a2df..d340eab5 100644 --- a/website/events/models.py +++ b/website/events/models.py @@ -301,8 +301,9 @@ class Event(models.Model, metaclass=ModelTranslateMeta): registration_reminder.category = Category.objects.get( key='event') registration_reminder.time = registration_reminder_time - registration_reminder.url = reverse("events:event", - args=[self.id]) + registration_reminder.url = ( + f'{settings.BASE_URL}' + f'{reverse("events:event", args=[self.id])}') registration_reminder.save() self.registration_reminder = registration_reminder @@ -321,10 +322,10 @@ class Event(models.Model, metaclass=ModelTranslateMeta): if start_reminder_time > timezone.now(): start_reminder.title_en = 'Event' start_reminder.title_nl = 'Evenement' - start_reminder.body_en = ('\'{}\' starts in ' - '1 hour'.format(self.title_en)) - start_reminder.body_nl = ('\'{}\' begint over ' - '1 uur'.format(self.title_nl)) + start_reminder.body_en = (f'\'{self.title_en}\' starts in ' + '1 hour') + start_reminder.body_nl = (f'\'{self.title_nl}\' begint over ' + '1 uur') start_reminder.category = Category.objects.get(key='event') start_reminder.time = start_reminder_time start_reminder.save() diff --git a/website/members/emails.py b/website/members/emails.py index aab345a4..08eaf056 100644 --- a/website/members/emails.py +++ b/website/members/emails.py @@ -142,7 +142,7 @@ def send_expiration_announcement(dry_run=False): settings.MEMBERSHIP_PRICES['year'], 2 ), 'renewal_url': '{}{}'.format( - 'https://thalia.nu', + settings.BASE_URL, reverse('registrations:renew') ) }) @@ -202,7 +202,7 @@ def send_email_change_confirmation_messages(change_request): 'members/email/email_change_confirm.txt', { 'confirm_link': '{}{}'.format( - 'https://thalia.nu', + settings.BASE_URL, reverse( 'members:email-change-confirm', args=[change_request.confirm_key] @@ -220,7 +220,7 @@ def send_email_change_confirmation_messages(change_request): 'members/email/email_change_verify.txt', { 'confirm_link': '{}{}'.format( - 'https://thalia.nu', + settings.BASE_URL, reverse( 'members:email-change-verify', args=[change_request.verify_key] diff --git a/website/registrations/emails.py b/website/registrations/emails.py index 42c9b23b..9f6e604a 100644 --- a/website/registrations/emails.py +++ b/website/registrations/emails.py @@ -24,7 +24,7 @@ def send_registration_email_confirmation(registration): { 'name': registration.get_full_name(), 'confirm_link': '{}{}'.format( - 'https://thalia.nu', + settings.BASE_URL, reverse('registrations:confirm-email', args=[registration.pk]) ) diff --git a/website/thaliawebsite/settings/settings.py b/website/thaliawebsite/settings/settings.py index f0037e5b..3a19d321 100644 --- a/website/thaliawebsite/settings/settings.py +++ b/website/thaliawebsite/settings/settings.py @@ -35,6 +35,7 @@ if not DEBUG: # Django 1.10.3 security release changed behaviour ALLOWED_HOSTS = [] SITE_ID = 1 +BASE_URL = 'https://thalia.nu' DATA_UPLOAD_MAX_NUMBER_FIELDS = 10000 # Useful for managing members -- GitLab From 43bf27e096ff8efa1ac7502fe9afd12ad9618e80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Se=CC=81bastiaan=20Versteeg?= Date: Sun, 11 Nov 2018 13:19:34 +0100 Subject: [PATCH 4/5] Fix tests --- docker-compose.yml | 3 --- ...13_auto_20181111_1238.py => 0013_auto_20181111_1319.py} | 7 ++++++- website/pushnotifications/models.py | 5 +++-- 3 files changed, 9 insertions(+), 6 deletions(-) rename website/pushnotifications/migrations/{0013_auto_20181111_1238.py => 0013_auto_20181111_1319.py} (69%) diff --git a/docker-compose.yml b/docker-compose.yml index adb96849..ff84a9bd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,6 @@ services: - 8000:8000 depends_on: - postgres - - redis volumes: - ./website:/usr/src/app/website/ - concrexit:/concrexit/ @@ -27,5 +26,3 @@ services: volumes: concrexit: driver: local - redis: - driver: local diff --git a/website/pushnotifications/migrations/0013_auto_20181111_1238.py b/website/pushnotifications/migrations/0013_auto_20181111_1319.py similarity index 69% rename from website/pushnotifications/migrations/0013_auto_20181111_1238.py rename to website/pushnotifications/migrations/0013_auto_20181111_1319.py index a9fa1d74..1d31dceb 100644 --- a/website/pushnotifications/migrations/0013_auto_20181111_1238.py +++ b/website/pushnotifications/migrations/0013_auto_20181111_1319.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.3 on 2018-11-11 11:38 +# Generated by Django 2.1.3 on 2018-11-11 12:19 from django.db import migrations, models @@ -19,4 +19,9 @@ class Migration(migrations.Migration): name='executed', field=models.DateTimeField(null=True), ), + migrations.AddField( + model_name='scheduledmessage', + name='scheduled', + field=models.BooleanField(default=True), + ), ] diff --git a/website/pushnotifications/models.py b/website/pushnotifications/models.py index f0fc67e6..30bc5cf8 100755 --- a/website/pushnotifications/models.py +++ b/website/pushnotifications/models.py @@ -77,7 +77,7 @@ class MessageManager(models.Manager): def get_queryset(self): return (super().get_queryset() - .filter(scheduledmessage__task_id=None)) + .filter(scheduledmessage__scheduled=None)) class Message(models.Model, metaclass=ModelTranslateMeta): @@ -164,7 +164,7 @@ class Message(models.Model, metaclass=ModelTranslateMeta): for reg_id in reg_ids: message.token = reg_id try: - messaging.send(message,dry_run=kwargs.get( + messaging.send(message, dry_run=kwargs.get( 'dry_run', False)) success_total += 1 except messaging.ApiCallError as e: @@ -196,5 +196,6 @@ class ScheduledMessage(Message, metaclass=ModelTranslateMeta): objects = ScheduledMessageManager() + scheduled = models.BooleanField(default=True) time = models.DateTimeField() executed = models.DateTimeField(null=True) -- GitLab From 71f0442445b70e83e8e04645fe8f4db8ff49335e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Se=CC=81bastiaan=20Versteeg?= Date: Thu, 22 Nov 2018 17:49:30 +0100 Subject: [PATCH 5/5] Disable firebase init when no settings are set --- website/thaliawebsite/settings/__init__.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/website/thaliawebsite/settings/__init__.py b/website/thaliawebsite/settings/__init__.py index 05008b64..50fc3b05 100644 --- a/website/thaliawebsite/settings/__init__.py +++ b/website/thaliawebsite/settings/__init__.py @@ -33,8 +33,9 @@ if os.environ.get('DJANGO_PRODUCTION'): # pragma: nocover if os.environ.get('GITLAB_CI'): # pragma: nocover from .testing import * # pylint: disable=wildcard-import -try: - initialize_app( - credential=credentials.Certificate(FIREBASE_CREDENTIALS)) -except ValueError as e: - logger.error('Firebase application failed to initialise') +if FIREBASE_CREDENTIALS != {}: + try: + initialize_app( + credential=credentials.Certificate(FIREBASE_CREDENTIALS)) + except ValueError as e: + logger.error('Firebase application failed to initialise') -- GitLab