Basic Google Groups syncing from concrexit

parent 6dd4e481
This diff is collapsed.
......@@ -31,6 +31,7 @@ django-sendfile2 = "~0.4.2"
# docs requirements
recommonmark = { version = "~0.6.0", optional = true }
sphinx = { version = "~2.2", optional = true }
google-api-python-client = "^1.7.11"
[tool.poetry.extras]
docs = ["recommonmark", "sphinx"]
......
......@@ -28,5 +28,5 @@ class MailingListAdmin(admin.ModelAdmin):
def alias_names(self, obj):
"""Return list of aliases of obj."""
return [x.alias for x in obj.aliasses.all()]
alias_names.short_description = _('List aliasses')
return [x.alias for x in obj.aliases.all()]
alias_names.short_description = _('List aliases')
......@@ -19,7 +19,7 @@ class MailingListSerializer(serializers.ModelSerializer):
def _names(self, instance):
"""Return list of names of the the mailing list and its aliases."""
return [instance.name] + [x.alias for x in instance.aliasses.all()]
return [instance.name] + [x.alias for x in instance.aliases.all()]
def _addresses(self, instance):
"""Return list of all subscribed addresses."""
......
......@@ -8,3 +8,7 @@ class MailinglistsConfig(AppConfig):
name = 'mailinglists'
verbose_name = _('Mailing lists')
def ready(self):
"""Imports the signals when the app is ready"""
from . import signals # noqa: F401
"""GSuite syncing helpers defined by the mailinglists package"""
from time import sleep
from django.utils.datastructures import ImmutableList
from googleapiclient.errors import HttpError
from mailinglists.models import MailingList
from mailinglists.services import get_automatic_lists
from utils.google_api import get_directory_api
from django.conf import settings
class GroupData:
def __init__(self, name, description, moderated=False, archived=False,
prefix='', aliases=ImmutableList([]),
addresses=ImmutableList([])):
super().__init__()
self.moderated = moderated
self.archived = archived
self.prefix = prefix
self.name = name
self.description = description
self.aliases = aliases
self.addresses = addresses
def create_group(group):
"""
Create a new group based on the provided data
:param group: group data
"""
directory_api = get_directory_api()
directory_api.groups().insert(
body={
'email': f'{group.name}@{settings.GSUITE_DOMAIN}',
'name': group.name,
'description': group.description,
},
).execute()
# Wait for mailinglist creation to complete
# Docs say we need to wait a minute, but since we always update lists
# an error in the list members update is not a problem
sleep(0.5)
update_group_members(group)
def update_group(old_name, group):
"""
Update a group based on the provided name and data
:param old_name: old group name
:param group: new group data
"""
directory_api = get_directory_api()
directory_api.groups().update(
groupKey=f'{old_name}@{settings.GSUITE_DOMAIN}',
body={
'email': f'{group.name}@{settings.GSUITE_DOMAIN}',
'name': group.name,
'description': group.description,
}
)
update_group_aliases(group)
update_group_members(group)
def update_group_aliases(group: GroupData):
"""
Update the aliases of a group based on existing values
:param group: group data
"""
directory_api = get_directory_api()
aliases_response = directory_api.groups().aliases().list(
groupKey=f'{group.name}@{settings.GSUITE_DOMAIN}',
).execute()
existing_aliases = [a['alias'] for a in
aliases_response.get('aliases', [])]
new_aliases = [f'{a}@{settings.GSUITE_DOMAIN}' for a in group.aliases]
remove_list = [x for x in existing_aliases if x not in new_aliases]
insert_list = [x for x in new_aliases if x not in existing_aliases]
for remove_alias in remove_list:
try:
directory_api.groups().aliases().delete(
groupKey=f'{group.name}@{settings.GSUITE_DOMAIN}',
alias=remove_alias
).execute()
except HttpError:
pass # Ignore error, API returned failing value
for insert_alias in insert_list:
try:
directory_api.groups().aliases().insert(
groupKey=f'{group.name}@{settings.GSUITE_DOMAIN}',
body={
'alias': insert_alias
}
).execute()
except HttpError:
pass # Ignore error, API returned failing value
def delete_group(group: GroupData):
"""
Removes the specified list from
:param group: group data
"""
directory_api = get_directory_api()
directory_api.groups().delete(
groupKey=f'{group.name}@{settings.GSUITE_DOMAIN}',
).execute()
def update_group_members(group: GroupData):
"""
Update the group members of the specified group based
on the existing members
:param group: group data
"""
directory_api = get_directory_api()
try:
members_response = directory_api.members().list(
groupKey=f'{group.name}@{settings.GSUITE_DOMAIN}',
).execute()
members_list = members_response.get('members', [])
while 'nextPageToken' in members_response:
members_response = directory_api.members().list(
groupKey=f'{group.name}@{settings.GSUITE_DOMAIN}',
pageToken=members_response['nextPageToken']
).execute()
members_list += members_response.get('members', [])
existing_members = [m['email'] for m in members_list]
except HttpError:
return # the list does not exist or something else is wrong
new_members = list(group.addresses)
remove_list = [x for x in existing_members if x not in new_members]
insert_list = [x for x in new_members if x not in existing_members]
for remove_member in remove_list:
try:
directory_api.members().delete(
groupKey=f'{group.name}@{settings.GSUITE_DOMAIN}',
memberKey=remove_member
).execute()
except HttpError:
pass # Ignore error, API returned failing value
for insert_member in insert_list:
try:
directory_api.members().insert(
groupKey=f'{group.name}@{settings.GSUITE_DOMAIN}',
body={
'email': insert_member,
'role': 'MEMBER'
}
).execute()
except HttpError:
pass # Ignore error, API returned failing value
def mailinglist_to_group(mailinglist: MailingList) -> GroupData:
"""Convert a mailinglist model to everything we need for GSuite"""
return GroupData(
moderated=mailinglist.moderated,
archived=mailinglist.archived,
prefix=mailinglist.prefix,
name=mailinglist.name,
description=mailinglist.description,
aliases=[x.alias for x in mailinglist.aliases.all()],
addresses=mailinglist.all_addresses()
)
def _automatic_to_group(automatic_list) -> GroupData:
"""Convert an automatic mailinglist to a GSuite Group data obj"""
return GroupData(
moderated=automatic_list['moderated'],
archived=automatic_list['archived'],
prefix=automatic_list['prefix'],
name=automatic_list['name'],
description=automatic_list['description'],
aliases=automatic_list['aliases'],
addresses=automatic_list['addresses']
)
def sync_mailinglists(lists=None):
"""
Sync mailing lists with GSuite
:param lists: optional parameter to determine which lists to sync
"""
directory_api = get_directory_api()
if lists is None:
lists = [
mailinglist_to_group(l) for l in MailingList.objects.all()
] + [
_automatic_to_group(l) for l in
get_automatic_lists()
]
try:
groups_response = directory_api.groups().list(
domain=settings.GSUITE_DOMAIN
).execute()
groups_list = groups_response.get('groups', [])
while 'nextPageToken' in groups_response:
groups_response = directory_api.groups().list(
domain=settings.GSUITE_DOMAIN,
pageToken=groups_response['nextPageToken']
).execute()
groups_list += groups_response.get('groups', [])
existing_groups = [g['name'] for g in groups_list]
except HttpError:
return # there are no lists or something went wrong
new_groups = [g.name for g in lists]
remove_list = [x for x in existing_groups if x not in new_groups]
insert_list = [x for x in new_groups if x not in existing_groups]
for l in lists:
if l.name in remove_list:
delete_group(l)
elif l.name in insert_list:
create_group(l)
else:
update_group(l.name, l)
"""Mailing list syncing management command"""
import logging
from django.core.management.base import BaseCommand
from mailinglists import gsuite
logger = logging.getLogger(__name__)
class Command(BaseCommand):
def handle(self, *args, **options):
"""Sync all mailing lists"""
gsuite.sync_mailinglists()
# Generated by Django 2.2.1 on 2019-10-13 21:52
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('mailinglists', '0014_mailinglist_description'),
]
operations = [
migrations.AlterModelOptions(
name='listalias',
options={'verbose_name': 'List alias', 'verbose_name_plural': 'List aliases'},
),
migrations.RemoveField(
model_name='mailinglist',
name='autoresponse_enabled',
),
migrations.RemoveField(
model_name='mailinglist',
name='autoresponse_text',
),
migrations.AlterField(
model_name='listalias',
name='mailinglist',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='aliases', to='mailinglists.MailingList', verbose_name='Mailing list'),
),
]
......@@ -80,18 +80,6 @@ class MailingList(models.Model):
blank=True,
)
autoresponse_enabled = models.BooleanField(
verbose_name=_("Automatic response enabled"),
default=False,
help_text=_('Indicate whether emails will get an automatic response.')
)
autoresponse_text = models.TextField(
verbose_name=_("Autoresponse text"),
null=True,
blank=True,
)
def all_addresses(self):
"""Return all addresses subscribed to this mailing list."""
for member in self.members.all():
......@@ -119,11 +107,6 @@ class MailingList(models.Model):
}
})
if not self.autoresponse_text and self.autoresponse_enabled:
raise ValidationError({
'autoresponse_text': _('Enter a text for the auto response.')
})
def __str__(self):
"""Return the name of the mailing list."""
return self.name
......@@ -169,7 +152,7 @@ class ListAlias(models.Model):
mailinglist = models.ForeignKey(MailingList,
verbose_name=_("Mailing list"),
on_delete=models.CASCADE,
related_name='aliasses')
related_name='aliases')
def clean(self):
"""Validate the alias."""
......@@ -195,4 +178,4 @@ class ListAlias(models.Model):
"""Meta class for ListAlias."""
verbose_name = _("List alias")
verbose_name_plural = _("List aliasses")
verbose_name_plural = _("List aliases")
"""The services defined by the mailinglists package"""
from django.conf import settings
from django.utils import timezone
......@@ -45,15 +46,15 @@ def get_automatic_lists():
lists = []
lists += _create_automatic_list(
['leden', 'members'], '[THALIA]',
['members', 'leden'], '[THALIA]',
Member.all_with_membership('member'), True, True, True)
lists += _create_automatic_list(
['begunstigers', 'benefactors'],
['benefactors', 'begunstigers'],
'[THALIA]',
Member.all_with_membership(Membership.BENEFACTOR),
multilingual=True)
lists += _create_automatic_list(
['ereleden', 'honorary'], '[THALIA]', Member.all_with_membership(
['honorary', 'ereleden'], '[THALIA]', Member.all_with_membership(
'honorary'), multilingual=True)
lists += _create_automatic_list(
['mentors'], '[THALIA] [MENTORS]', mentors, moderated=False)
......@@ -61,10 +62,10 @@ def get_automatic_lists():
['activemembers'], '[THALIA] [COMMITTEES]',
active_members)
lists += _create_automatic_list(
['commissievoorzitters', 'committeechairs'], '[THALIA] [CHAIRS]',
['committeechairs', 'commissievoorzitters'], '[THALIA] [CHAIRS]',
committee_chair_emails, moderated=False)
lists += _create_automatic_list(
['gezelschapvoorzitters', 'societychairs'], '[THALIA] [SOCIETY]',
['societychairs', 'gezelschapvoorzitters'], '[THALIA] [SOCIETY]',
society_chair_emails, moderated=False)
lists += _create_automatic_list(
['optin'], '[THALIA] [OPTIN]', Member.current_members.filter(
......@@ -74,7 +75,7 @@ def get_automatic_lists():
all_previous_board_members = []
for board in Board.objects.filter(
since__year__lte=lectureyear).order_by('since__year'):
since__year__lte=lectureyear).order_by('since__year'):
board_members = [board.member for board in
MemberGroupMembership.objects.filter
(group=board).prefetch_related('member')]
......@@ -97,10 +98,13 @@ def get_automatic_lists():
return lists
def _create_automatic_list(names, prefix, members,
def _create_automatic_list(names, prefix, members, description='',
archived=True, moderated=True, multilingual=False):
data = {
'names': names,
'name': names[0],
'description': description,
'aliases': names[1:],
'prefix': prefix,
'archived': archived,
'moderated': moderated,
......@@ -116,6 +120,8 @@ def _create_automatic_list(names, prefix, members,
if member.profile.language == language[0]]
localized_data['names'] = [
'{}-{}'.format(n, language[0]) for n in names]
localized_data['name'] = localized_data['names'][0]
localized_data['aliases'] = localized_data['names'][1:]
yield localized_data # these are localized lists, e.g. leden-nl@
else:
data['addresses'] = set([member.email for member in members])
......
from django.db.models.signals import post_save, post_delete
from django.dispatch import receiver
from mailinglists.gsuite import mailinglist_to_group, sync_mailinglists
@receiver(post_save, sender='mailinglists.MailingList')
def post_mailinglist_save(instance, **kwargs):
sync_mailinglists([mailinglist_to_group(instance)])
@receiver(post_delete, sender='mailinglists.MailingList')
def post_mailinglist_delete(instance, **kwargs):
sync_mailinglists([mailinglist_to_group(instance)])
......@@ -9,6 +9,8 @@ overrides.
# flake8: noqa: ignore F403
import logging
from firebase_admin import initialize_app, credentials
from googleapiclient.discovery import build
from google.oauth2 import service_account
# Load all default settings because we need to use settings.configure
# for sphinx documentation generation.
......@@ -39,3 +41,10 @@ if FIREBASE_CREDENTIALS != {}:
credential=credentials.Certificate(FIREBASE_CREDENTIALS))
except ValueError as e:
logger.error('Firebase application failed to initialise')
GSUITE_ADMIN_CREDENTIALS = (
service_account.Credentials.from_service_account_info(
GSUITE_ADMIN_CREDENTIALS, scopes=GSUITE_ADMIN_SCOPES
).with_subject(GSUITE_ADMIN_USER)
)
......@@ -78,6 +78,14 @@ if not (FIREBASE_CREDENTIALS == '{}'):
FIREBASE_CREDENTIALS = base64.urlsafe_b64decode(FIREBASE_CREDENTIALS)
FIREBASE_CREDENTIALS = json.loads(FIREBASE_CREDENTIALS)
GSUITE_ADMIN_CREDENTIALS = os.environ.get('GSUITE_ADMIN_CREDENTIALS', '{}')
if not (GSUITE_ADMIN_CREDENTIALS == '{}'):
GSUITE_ADMIN_CREDENTIALS = base64.urlsafe_b64decode(
GSUITE_ADMIN_CREDENTIALS)
GSUITE_ADMIN_CREDENTIALS = json.loads(GSUITE_ADMIN_CREDENTIALS)
GSUITE_ADMIN_USER = os.environ.get('GSUITE_ADMIN_USER', None)
GSUITE_DOMAIN = os.environ.get('GSUITE_DOMAIN', 'thalia.nu')
if os.environ.get('DJANGO_SSLONLY'):
SECURE_SSL_REDIRECT = True
SESSION_COOKIE_SECURE = True
......
......@@ -261,6 +261,16 @@ THUMBNAIL_SIZES = {
# Placeholder Firebase config
FIREBASE_CREDENTIALS = {}
# Placeholder GSuite config
GSUITE_ADMIN_CREDENTIALS = {}
GSUITE_ADMIN_USER = 'concrexit@thalia.nu'
GSUITE_ADMIN_SCOPES = [
'https://www.googleapis.com/auth/admin.directory.group',
'https://www.googleapis.com/auth/admin.directory.user',
'https://www.googleapis.com/auth/apps.groups.settings'
]
GSUITE_DOMAIN = 'thalia.localhost'
# Default FROM email
DEFAULT_FROM_EMAIL = f'noreply@{SITE_DOMAIN}'
SERVER_EMAIL = DEFAULT_FROM_EMAIL
......
from googleapiclient.discovery import build
from googleapiclient.discovery_cache.base import Cache
from thaliawebsite import settings
class MemoryCache(Cache):
_CACHE = {}
def get(self, url):
return MemoryCache._CACHE.get(url)
def set(self, url, content):
MemoryCache._CACHE[url] = content
memory_cache = MemoryCache()
def get_directory_api():
return build(
'admin', 'directory_v1',
credentials=settings.GSUITE_ADMIN_CREDENTIALS,
cache=memory_cache
)
def get_group_settings_api():
return build(
'groups', 'v1',
credentials=settings.GSUITE_ADMIN_CREDENTIALS,
cache=memory_cache
)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment