Merge branch 'feature/migration-docs-keys' into 'master'

Update migration docs and normalize API keys

Documentatie voor migratie toegevoegd, scripts makkelijker gemaakt en de keys allemaal hetzelfde gemaakt.

Zie ook thalia/website!590, die moet eerst

See merge request !128

migration.md

@@ -11,11 +11,21 @@ This is taken care of during member migration.
 
 ## Documents
 
+Make sure you have valid login credentials for thalia.nu.
+
+To migrate all documents execute `python manage.py migratedocuments`
+and enter the required login information.
+
 ## Education
 
+Make sure that the API key is available as `MIGRATION_KEY` in settings.py.
+
+To migrate the events, make sure that membershave been migrated.
+Then simply execute `python manage.py migrateeducation`.
+
 ## Events
 
-Make sure that the API key is available as `EVENTS_MIGRATION_KEY` in settings.py.
+Make sure that the API key is available as `MIGRATION_KEY` in settings.py.
 
 To migrate the events, make sure that members, committees and boards
 have been migrated. Then simply execute `python manage.py migrateevents`.
@@ -49,3 +59,8 @@ This migration typically takes a few minutes.
 ## Thabloids
 
 ## Merchandise
+
+Make sure you have valid login credentials for thalia.nu.
+
+To migrate the merchandise execute `python manage.py migratemerchandise` and
+enter the required login information.
\ No newline at end of file

website/documents/management/commands/migratemeetings.py → website/documents/management/commands/migratedocuments.py

@@ -3,7 +3,12 @@ from django.utils import timezone
 from django.utils.timezone import datetime
 
 from utils.management.commands import legacylogin
-from documents.models import GeneralMeeting, GeneralMeetingDocument
+from documents.models import (
+    GeneralMeeting,
+    GeneralMeetingDocument,
+    AssociationDocumentsYear,
+    MiscellaneousDocument
+)
 
 from bs4 import BeautifulSoup
 import requests
@@ -16,12 +21,15 @@ def filefield_from_url(filefield, url):
 
 
 class Command(legacylogin.Command):
-    help = "Scrapes the General Meetings from the old Thalia website"
+    help = "Scrapes the documents from the old Thalia website"
 
     def handle(self, *args, **options):
         super().handle(*args, **options)
+
+        print("Migrating the general meetings")
         url = "https://thalia.nu/ajax/alvyearview?year={}"
         for year in range(1990, 2016):
+            print("Migrating {}".format(year))
             src = self.session.get(url.format(year)).text
             soup = BeautifulSoup(src, 'lxml')
             for alv in soup.div.find_all('div', recursive=False):
@@ -46,3 +54,46 @@ class Command(legacylogin.Command):
                     doc.meeting = meeting
                     filefield_from_url(doc.file, doc_url)
                     doc.save()
+        print("Migrating general meetings complete")
+
+        print("Migrating the policy documents")
+        url = "https://thalia.nu/association/documents"
+        documentpage = self.session.get(url)
+        soup = BeautifulSoup(documentpage.text, 'lxml')
+        wrapper = soup(attrs={'class': 'policywrapper'})[0]
+        uls = wrapper.find_all('ul', recursive=False)
+        policies = uls[0].find_all('li', recursive=False)
+        reports = uls[1].find_all('li', recursive=False)
+        for policy, report in zip(policies, reports):
+            year = policy.find('h2').find(text=True)
+            print("Migrating {}".format(year))
+            year = int(year.replace("Beleidsplan '", '')[:2])
+            year += (19 if year >= 90 else 20) * 100
+            obj, cr = AssociationDocumentsYear.objects.get_or_create(year=year)
+            obj.year = year
+            files = [(obj.policy_document,
+                      policy.find(attrs={'class': 'overlay-icon-link'})),
+                     (obj.annual_report,
+                      report.find(attrs={'class': 'overlay-icon-link'})),
+                     (obj.financial_report,
+                      report.find(attrs={'class': 'overlay-icon-euro'}))]
+            for filefield, url in files:
+                if url is not None:
+                    url = "https://thalia.nu" + url['href']
+                    file = ContentFile(requests.get(url).content)
+                    # File names are ignored when serving files anyway
+                    filefield.save(os.path.basename(url), file)
+        print("Migrating policy documents complete")
+
+        print("Migrating the miscellaneous documents")
+        container = soup(attrs={'class': 'generalcontainer'})[0]
+        documents = container.find_all('li', recursive=False)
+        for document in documents:
+            name = document.find('h2').find(text=True)
+            print("Migrating {}".format(name))
+            obj, cr = MiscellaneousDocument.objects.get_or_create(name=name)
+            url = document.find(attrs={'class': 'overlay-icon-link'})
+            if url is not None:
+                url = "https://thalia.nu" + url['href']
+                filefield_from_url(obj.file, url)
+        print("Migrating miscellaneous documents complete")

website/documents/management/commands/migratemisc.py

-import os
-
-import requests
-from bs4 import BeautifulSoup
-from django.core.files.base import ContentFile
-from documents.models import MiscellaneousDocument
-from utils.management.commands import legacylogin
-
-
-def filefield_from_url(filefield, url):
-    file = ContentFile(requests.get(url).content)
-    filefield.save(os.path.basename(url), file)
-
-
-class Command(legacylogin.Command):
-    help = "Scrapes the Miscellaneous Documents from the old Thalia website"
-
-    def handle(self, *args, **options):
-        super().handle(*args, **options)
-        url = "https://thalia.nu/association/documents"
-        documentpage = self.session.get(url)
-        soup = BeautifulSoup(documentpage.text, 'lxml')
-        container = soup(attrs={'class': 'generalcontainer'})[0]
-        documents = container.find_all('li', recursive=False)
-        for document in documents:
-            name = document.find('h2').find(text=True)
-            obj, cr = MiscellaneousDocument.objects.get_or_create(name=name)
-            url = document.find(attrs={'class': 'overlay-icon-link'})
-            if url is not None:
-                url = "https://thalia.nu" + url['href']
-                filefield_from_url(obj.file, url)

website/documents/management/commands/migratepolicydocuments.py

-from django.core.files.base import ContentFile
-
-from utils.management.commands import legacylogin
-from documents.models import AssociationDocumentsYear
-
-from bs4 import BeautifulSoup
-import requests
-import os
-
-
-class Command(legacylogin.Command):
-    help = "Scrapes the policy documents from the old Thalia website"
-
-    def handle(self, *args, **options):
-        super().handle(*args, **options)
-        url = "https://thalia.nu/association/documents"
-        documentpage = self.session.get(url)
-        soup = BeautifulSoup(documentpage.text, 'lxml')
-        wrapper = soup(attrs={'class': 'policywrapper'})[0]
-        uls = wrapper.find_all('ul', recursive=False)
-        policies = uls[0].find_all('li', recursive=False)
-        reports = uls[1].find_all('li', recursive=False)
-        for policy, report in zip(policies, reports):
-            year = policy.find('h2').find(text=True)
-            year = int(year.replace("Beleidsplan '", '')[:2])
-            year = (19 if year >= 90 else 20)*100 + year
-            obj, cr = AssociationDocumentsYear.objects.get_or_create(year=year)
-            obj.year = year
-            files = [(obj.policy_document,
-                      policy.find(attrs={'class': 'overlay-icon-link'})),
-                     (obj.annual_report,
-                      report.find(attrs={'class': 'overlay-icon-link'})),
-                     (obj.financial_report,
-                      report.find(attrs={'class': 'overlay-icon-euro'}))]
-            for filefield, url in files:
-                if url is not None:
-                    url = "https://thalia.nu" + url['href']
-                    file = ContentFile(requests.get(url).content)
-                    # File names are ignored when serving files anyway
-                    filefield.save(os.path.basename(url), file)

website/education/management/commands/migrateeducation.py

@@ -2,8 +2,9 @@ import json
 import os
 
 import requests
+from django.conf import settings
 from django.contrib.auth.models import User
-from django.core.exceptions import PermissionDenied
+from django.core.exceptions import PermissionDenied, ImproperlyConfigured
 from django.core.files.base import ContentFile
 from django.utils.dateparse import parse_date
 from django.utils.translation import activate
@@ -22,6 +23,12 @@ class Command(legacylogin.Command):
     def handle(self, *args, **options):
         activate('en')
 
+        if not settings.MIGRATION_KEY:
+            raise ImproperlyConfigured("MIGRATION_KEY not specified")
+        url = "https://thalia.nu/index.php/onderwijs/api?apikey={}".format(
+            settings.MIGRATION_KEY
+        )
+
         input_val = input(
             'Do you want to delete all existing objects? (type yes or no) ')
         if input_val == 'yes':
@@ -31,8 +38,6 @@ class Command(legacylogin.Command):
             Category.objects.all().delete()
 
         session = requests.Session()
-        key = input('Please enter the education API key: ')
-        url = ('https://thalia.nu/index.php/onderwijs/api?apikey=' + key)
         src = session.get(url).text
 
         if 'invalid api key' in src:

website/events/management/commands/migrateevents.py

@@ -32,11 +32,11 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):
 
-        if not settings.EVENTS_MIGRATION_KEY:
-            raise ImproperlyConfigured('EVENTS_MIGRATION_KEY not specified')
+        if not settings.MIGRATION_KEY:
+            raise ImproperlyConfigured('MIGRATION_KEY not specified')
 
         events_api_url = 'https://thalia.nu/events/api/?apikey={}'.format(
-            settings.EVENTS_MIGRATION_KEY)
+            settings.MIGRATION_KEY)
 
         print('[*]Getting events json data')
         try: