Commit 8aacf7db authored by Joost Rijneveld's avatar Joost Rijneveld
Browse files

Merge branch 'feature/migration-docs-keys' into 'master'

Update migration docs and normalize API keys

Documentatie voor migratie toegevoegd, scripts makkelijker gemaakt en de keys allemaal hetzelfde gemaakt.

Zie ook thalia/website!590, die moet eerst

See merge request !128
parents 77f22fd9 be9e8043
......@@ -11,11 +11,21 @@ This is taken care of during member migration.
## Documents
Make sure you have valid login credentials for thalia.nu.
To migrate all documents execute `python manage.py migratedocuments`
and enter the required login information.
## Education
Make sure that the API key is available as `MIGRATION_KEY` in settings.py.
To migrate the events, make sure that membershave been migrated.
Then simply execute `python manage.py migrateeducation`.
## Events
Make sure that the API key is available as `EVENTS_MIGRATION_KEY` in settings.py.
Make sure that the API key is available as `MIGRATION_KEY` in settings.py.
To migrate the events, make sure that members, committees and boards
have been migrated. Then simply execute `python manage.py migrateevents`.
......@@ -49,3 +59,8 @@ This migration typically takes a few minutes.
## Thabloids
## Merchandise
Make sure you have valid login credentials for thalia.nu.
To migrate the merchandise execute `python manage.py migratemerchandise` and
enter the required login information.
\ No newline at end of file
......@@ -3,7 +3,12 @@ from django.utils import timezone
from django.utils.timezone import datetime
from utils.management.commands import legacylogin
from documents.models import GeneralMeeting, GeneralMeetingDocument
from documents.models import (
GeneralMeeting,
GeneralMeetingDocument,
AssociationDocumentsYear,
MiscellaneousDocument
)
from bs4 import BeautifulSoup
import requests
......@@ -16,12 +21,15 @@ def filefield_from_url(filefield, url):
class Command(legacylogin.Command):
help = "Scrapes the General Meetings from the old Thalia website"
help = "Scrapes the documents from the old Thalia website"
def handle(self, *args, **options):
super().handle(*args, **options)
print("Migrating the general meetings")
url = "https://thalia.nu/ajax/alvyearview?year={}"
for year in range(1990, 2016):
print("Migrating {}".format(year))
src = self.session.get(url.format(year)).text
soup = BeautifulSoup(src, 'lxml')
for alv in soup.div.find_all('div', recursive=False):
......@@ -46,3 +54,46 @@ class Command(legacylogin.Command):
doc.meeting = meeting
filefield_from_url(doc.file, doc_url)
doc.save()
print("Migrating general meetings complete")
print("Migrating the policy documents")
url = "https://thalia.nu/association/documents"
documentpage = self.session.get(url)
soup = BeautifulSoup(documentpage.text, 'lxml')
wrapper = soup(attrs={'class': 'policywrapper'})[0]
uls = wrapper.find_all('ul', recursive=False)
policies = uls[0].find_all('li', recursive=False)
reports = uls[1].find_all('li', recursive=False)
for policy, report in zip(policies, reports):
year = policy.find('h2').find(text=True)
print("Migrating {}".format(year))
year = int(year.replace("Beleidsplan '", '')[:2])
year += (19 if year >= 90 else 20) * 100
obj, cr = AssociationDocumentsYear.objects.get_or_create(year=year)
obj.year = year
files = [(obj.policy_document,
policy.find(attrs={'class': 'overlay-icon-link'})),
(obj.annual_report,
report.find(attrs={'class': 'overlay-icon-link'})),
(obj.financial_report,
report.find(attrs={'class': 'overlay-icon-euro'}))]
for filefield, url in files:
if url is not None:
url = "https://thalia.nu" + url['href']
file = ContentFile(requests.get(url).content)
# File names are ignored when serving files anyway
filefield.save(os.path.basename(url), file)
print("Migrating policy documents complete")
print("Migrating the miscellaneous documents")
container = soup(attrs={'class': 'generalcontainer'})[0]
documents = container.find_all('li', recursive=False)
for document in documents:
name = document.find('h2').find(text=True)
print("Migrating {}".format(name))
obj, cr = MiscellaneousDocument.objects.get_or_create(name=name)
url = document.find(attrs={'class': 'overlay-icon-link'})
if url is not None:
url = "https://thalia.nu" + url['href']
filefield_from_url(obj.file, url)
print("Migrating miscellaneous documents complete")
import os
import requests
from bs4 import BeautifulSoup
from django.core.files.base import ContentFile
from documents.models import MiscellaneousDocument
from utils.management.commands import legacylogin
def filefield_from_url(filefield, url):
file = ContentFile(requests.get(url).content)
filefield.save(os.path.basename(url), file)
class Command(legacylogin.Command):
help = "Scrapes the Miscellaneous Documents from the old Thalia website"
def handle(self, *args, **options):
super().handle(*args, **options)
url = "https://thalia.nu/association/documents"
documentpage = self.session.get(url)
soup = BeautifulSoup(documentpage.text, 'lxml')
container = soup(attrs={'class': 'generalcontainer'})[0]
documents = container.find_all('li', recursive=False)
for document in documents:
name = document.find('h2').find(text=True)
obj, cr = MiscellaneousDocument.objects.get_or_create(name=name)
url = document.find(attrs={'class': 'overlay-icon-link'})
if url is not None:
url = "https://thalia.nu" + url['href']
filefield_from_url(obj.file, url)
from django.core.files.base import ContentFile
from utils.management.commands import legacylogin
from documents.models import AssociationDocumentsYear
from bs4 import BeautifulSoup
import requests
import os
class Command(legacylogin.Command):
help = "Scrapes the policy documents from the old Thalia website"
def handle(self, *args, **options):
super().handle(*args, **options)
url = "https://thalia.nu/association/documents"
documentpage = self.session.get(url)
soup = BeautifulSoup(documentpage.text, 'lxml')
wrapper = soup(attrs={'class': 'policywrapper'})[0]
uls = wrapper.find_all('ul', recursive=False)
policies = uls[0].find_all('li', recursive=False)
reports = uls[1].find_all('li', recursive=False)
for policy, report in zip(policies, reports):
year = policy.find('h2').find(text=True)
year = int(year.replace("Beleidsplan '", '')[:2])
year = (19 if year >= 90 else 20)*100 + year
obj, cr = AssociationDocumentsYear.objects.get_or_create(year=year)
obj.year = year
files = [(obj.policy_document,
policy.find(attrs={'class': 'overlay-icon-link'})),
(obj.annual_report,
report.find(attrs={'class': 'overlay-icon-link'})),
(obj.financial_report,
report.find(attrs={'class': 'overlay-icon-euro'}))]
for filefield, url in files:
if url is not None:
url = "https://thalia.nu" + url['href']
file = ContentFile(requests.get(url).content)
# File names are ignored when serving files anyway
filefield.save(os.path.basename(url), file)
......@@ -2,8 +2,9 @@ import json
import os
import requests
from django.conf import settings
from django.contrib.auth.models import User
from django.core.exceptions import PermissionDenied
from django.core.exceptions import PermissionDenied, ImproperlyConfigured
from django.core.files.base import ContentFile
from django.utils.dateparse import parse_date
from django.utils.translation import activate
......@@ -22,6 +23,12 @@ class Command(legacylogin.Command):
def handle(self, *args, **options):
activate('en')
if not settings.MIGRATION_KEY:
raise ImproperlyConfigured("MIGRATION_KEY not specified")
url = "https://thalia.nu/index.php/onderwijs/api?apikey={}".format(
settings.MIGRATION_KEY
)
input_val = input(
'Do you want to delete all existing objects? (type yes or no) ')
if input_val == 'yes':
......@@ -31,8 +38,6 @@ class Command(legacylogin.Command):
Category.objects.all().delete()
session = requests.Session()
key = input('Please enter the education API key: ')
url = ('https://thalia.nu/index.php/onderwijs/api?apikey=' + key)
src = session.get(url).text
if 'invalid api key' in src:
......
......@@ -32,11 +32,11 @@ class Command(BaseCommand):
def handle(self, *args, **options):
if not settings.EVENTS_MIGRATION_KEY:
raise ImproperlyConfigured('EVENTS_MIGRATION_KEY not specified')
if not settings.MIGRATION_KEY:
raise ImproperlyConfigured('MIGRATION_KEY not specified')
events_api_url = 'https://thalia.nu/events/api/?apikey={}'.format(
settings.EVENTS_MIGRATION_KEY)
settings.MIGRATION_KEY)
print('[*]Getting events json data')
try:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment