Commit 7fb6d3c3 authored by Joren Vrancken's avatar Joren Vrancken

Merge branch 'use-threadpool' into 'master'

Use ThreadPoolExecutor instead of unlimited threads for gsuite sync

See merge request !1394
parents 958592a3 f6c02ccd
"""GSuite syncing helpers defined by the mailinglists package""" """GSuite syncing helpers defined by the mailinglists package"""
import logging import logging
import threading from concurrent.futures import wait
from concurrent.futures.thread import ThreadPoolExecutor
from time import sleep from time import sleep
from typing import List from typing import List
...@@ -82,6 +83,7 @@ class GSuiteSyncService: ...@@ -82,6 +83,7 @@ class GSuiteSyncService:
groupUniqueId=f'{group.name}@{settings.GSUITE_DOMAIN}', groupUniqueId=f'{group.name}@{settings.GSUITE_DOMAIN}',
body=self._group_settings(group.moderated) body=self._group_settings(group.moderated)
).execute() ).execute()
logger.info(f'List {group.name} created')
except HttpError as e: except HttpError as e:
logger.error(f'Could not successfully finish ' logger.error(f'Could not successfully finish '
f'creating the list {group.name}', e.content) f'creating the list {group.name}', e.content)
...@@ -109,6 +111,7 @@ class GSuiteSyncService: ...@@ -109,6 +111,7 @@ class GSuiteSyncService:
groupUniqueId=f'{group.name}@{settings.GSUITE_DOMAIN}', groupUniqueId=f'{group.name}@{settings.GSUITE_DOMAIN}',
body=self._group_settings(group.moderated) body=self._group_settings(group.moderated)
).execute() ).execute()
logger.info(f'List {group.name} updated')
except HttpError as e: except HttpError as e:
logger.error(f'Could not update list {group.name}', e.content) logger.error(f'Could not update list {group.name}', e.content)
return return
...@@ -165,6 +168,8 @@ class GSuiteSyncService: ...@@ -165,6 +168,8 @@ class GSuiteSyncService:
e.content e.content
) )
logger.info(f'List {group.name} aliases updated')
def delete_group(self, name: str): def delete_group(self, name: str):
""" """
Set the specified list to unused, this is not a real delete Set the specified list to unused, this is not a real delete
...@@ -184,6 +189,7 @@ class GSuiteSyncService: ...@@ -184,6 +189,7 @@ class GSuiteSyncService:
self._update_group_aliases( self._update_group_aliases(
GSuiteSyncService.GroupData(name, aliases=[]) GSuiteSyncService.GroupData(name, aliases=[])
) )
logger.info(f'List {name} deleted')
except HttpError as e: except HttpError as e:
logger.error(f'Could not delete list {name}', e.content) logger.error(f'Could not delete list {name}', e.content)
...@@ -241,6 +247,8 @@ class GSuiteSyncService: ...@@ -241,6 +247,8 @@ class GSuiteSyncService:
logger.error(f'Could not insert list member ' logger.error(f'Could not insert list member '
f'{insert_member} in {group.name}', e.content) f'{insert_member} in {group.name}', e.content)
logger.info(f'List {group.name} members updated')
@staticmethod @staticmethod
def mailinglist_to_group(mailinglist: MailingList): def mailinglist_to_group(mailinglist: MailingList):
"""Convert a mailinglist model to everything we need for GSuite""" """Convert a mailinglist model to everything we need for GSuite"""
...@@ -302,25 +310,16 @@ class GSuiteSyncService: ...@@ -302,25 +310,16 @@ class GSuiteSyncService:
remove_list = [x for x in existing_groups if x not in new_groups] remove_list = [x for x in existing_groups if x not in new_groups]
insert_list = [x for x in new_groups if x not in existing_groups] insert_list = [x for x in new_groups if x not in existing_groups]
threads = [] executor = ThreadPoolExecutor(max_workers=4)
futures = []
for l in lists: for l in lists:
if l.name in insert_list and l.name not in archived_groups: if l.name in insert_list and l.name not in archived_groups:
thread = threading.Thread(target=self.create_group, futures.append(executor.submit(self.create_group, l))
args=(l,))
threads.append(thread)
thread.start()
elif len(l.addresses) > 0: elif len(l.addresses) > 0:
thread = threading.Thread(target=self.update_group, futures.append(executor.submit(self.update_group, l.name, l))
args=(l.name, l))
threads.append(thread)
thread.start()
for l in remove_list: for l in remove_list:
thread = threading.Thread(target=self.delete_group, futures.append(executor.submit(self.delete_group, l))
args=(l,))
threads.append(thread)
thread.start()
for th in threads: wait(futures)
th.join()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment