Commit 4b621377 authored by Joost Rijneveld's avatar Joost Rijneveld

Merge branch 'feature/import-newsletters' into 'master'

Import newsletters

Basic newsletter importing, not perfect but good enough.

Closes #97 

See merge request !177
parents b8eceaac 3f0c6fd8
......@@ -50,6 +50,10 @@ This migration typically takes a few minutes.
## Newsletters
Make sure that the API key is available as `MIGRATION_KEY` in settings.py.
To migrate the newsletters simply execute `python manage.py migratenewsletters`.
## Partners
To migrate partners, partner events, vacancy categories and vacancies,
......
import requests
import re
import dateparser
from bs4 import BeautifulSoup
from datetime import datetime, timedelta, date
from django.utils import timezone
from django.core.management.base import BaseCommand
from newsletters.models import Newsletter, NewsletterItem, NewsletterEvent
def isoweek_firstday(year, week):
ret = datetime.strptime('%04d-%02d-1' % (year, week), '%Y-%W-%w')
if date(year, 1, 4).isoweekday() > 4:
ret -= timedelta(days=7)
return ret
class Command(BaseCommand):
help = 'Scrapes the newsletters from the old Thalia website'
def handle(self, *args, **options):
year_range = range(2015, 2017)
week_range = range(1, 53)
url = "https://thalia.nu/nieuwsbrief/{}/{:02d}/nieuwsbrief.html"
NewsletterItem.objects.all().delete()
NewsletterEvent.objects.all().delete()
Newsletter.objects.all().delete()
session = requests.Session()
for year in year_range:
for week in week_range:
request = session.get(url.format(year, week))
src = request.text
if request.status_code != 200:
continue
soup = BeautifulSoup(src, "lxml")
newsletter = Newsletter()
title = "Newsletter week {} {}".format(week, year)
newsletter.title_nl = title
newsletter.title_en = title
newsletter.date = isoweek_firstday(year, week)
all_tr = soup.find("table").find_all("tr")
desc_contents = all_tr[2].find("td").contents
desc_td = "".join([str(x) for x in desc_contents])
newsletter.description_nl = desc_td
newsletter.description_en = desc_td
newsletter.sent = True
newsletter.save()
start = 6 if "AGENDA" in all_tr[3].text else 5
items = all_tr[start:len(all_tr)]
for index in range(int((len(items) - 2) / 2)):
first_index = index * 2
second_index = first_index + 1
title_el = items[first_index].find("td").find("h2")
title = title_el.text
content_tr = items[second_index]
content_td = content_tr.find_all("td")
description = "".join([str(x) for x in
content_td[0].contents])
item = None
if len(content_td) > 1:
item = NewsletterEvent()
event_data = content_td[1].find_all("span")
what = event_data[0].text
where = event_data[1].text
when = event_data[2].text
when = when.replace('vanaf', '')
parse_settings = {
'PREFER_DATES_FROM': 'future'
}
if "-" in when:
when = when.split("-", 2)
start_when = dateparser.parse(
when[0], settings=parse_settings)
end_when = dateparser.parse(
when[1], settings=parse_settings)
if len(when[1]) < 8:
end_when = end_when.replace(start_when.year,
start_when.month,
start_when.day)
else:
start_when = end_when = dateparser.parse(
when, settings=parse_settings)
tz = timezone.get_current_timezone()
if start_when is None and end_when is None:
start_when = end_when = newsletter.date
elif start_when is None:
start_when = end_when
elif end_when is None:
end_when = start_when
start_when = tz.localize(start_when)
end_when = tz.localize(end_when)
start_when = start_when.replace(year)
end_when = end_when.replace(year)
item.start_datetime = start_when
item.end_datetime = end_when
price = None
if ("PRIJS" in content_td[1].text and
event_data[3] is not None):
price = event_data[3].text
item.title_nl = item.what_nl = what[:len(what) - 1]
item.title_en = item.what_en = what[:len(what) - 1]
item.where_nl = where[:len(where) - 1]
item.where_en = where[:len(where) - 1]
if price is not None and "ratis" not in price:
price = price.replace(",", ".")
price = price.replace("-", "0")
if "\x82" in price:
price = price[3:len(price) - 1]
elif "euro" in price:
price = price[0:len(price) - 1]
price = price.replace("euro", "")
else:
price = price[1:len(price) - 1]
item.price = float(price)
costs = re.findall("Deze bedragen <b>€([0-9]*,"
"[0-9]*)\.</b>", description)
if len(costs) > 0:
item.penalty_costs = float(
costs[0][:len(costs[0]) - 1]
.replace(",", ".")
.replace("-", "0"))
elif item.price is not None:
item.penalty_costs = item.price
else:
item = NewsletterItem()
item.title_nl = title.lower().capitalize()
item.title_en = title.lower().capitalize()
item.description_nl = description
item.description_en = description
item.newsletter = newsletter
item.save()
# -*- coding: utf-8 -*-
# Generated by Django 1.10.2 on 2016-11-09 21:57
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('newsletters', '0001_initial'),
]
operations = [
migrations.AlterField(
model_name='newsletterevent',
name='price',
field=models.DecimalField(blank=True, decimal_places=2, default=None, max_digits=5, null=True, verbose_name='Price (in Euro)'),
),
]
# -*- coding: utf-8 -*-
# Generated by Django 1.10.2 on 2016-11-09 21:57
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('newsletters', '0002_auto_20161109_2257'),
]
operations = [
migrations.AlterField(
model_name='newsletterevent',
name='penalty_costs',
field=models.DecimalField(blank=True, decimal_places=2, default=None, help_text='This is the price that a member has to pay when he/she did not show up.', max_digits=5, null=True, verbose_name='Costs (in Euro)'),
),
]
......@@ -102,14 +102,18 @@ class NewsletterEvent(NewsletterContent):
verbose_name=_('Price (in Euro)'),
max_digits=5,
decimal_places=2,
default=0,
blank=True,
null=True,
default=None,
)
penalty_costs = models.DecimalField(
verbose_name=_('Costs (in Euro)'),
max_digits=5,
decimal_places=2,
default=0,
blank=True,
null=True,
default=None,
help_text=_('This is the price that a member has to '
'pay when he/she did not show up.'),
)
......
......@@ -112,7 +112,7 @@
<p style="font-family: Arial, sans-serif">
<i><b>{% trans "Attention" %}:</b> {% trans "Registration deadline = unregistration deadline" %}!
{% trans "Thalia will recover the costs on you if you do not unregister on time" %}.
{% if event.penalty_costs != 0 %}
{% if event.penalty_costs is not None and event.penalty_costs > 0 %}
{% trans "These costs are" %}: <b>€{{ event.penalty_costs }}.</b>
{% endif %}
</i>
......@@ -148,7 +148,7 @@
</b>
<br>
<span style="font-family: Arial, sans-serif; font-size: 15px; color: white; ">
{% if event.start_datetime == event.end_datetime %}
{% if event.start_datetime|date:'d-M-Y H:i' == event.end_datetime|date:'d-M-Y H:i' %}
{{ event.start_datetime }}
{% elif event.start_datetime|date:'d-M-Y' == event.end_datetime|date:'d-M-Y' %}
{{ event.start_datetime }} - {{ event.end_datetime|date:'H:i' }}
......@@ -158,6 +158,7 @@
</span>
</div>
{% if event.price is not None %}
<div style="margin-bottom: 5px;">
<b style="font-family: Calibri, 'Trebuchet MS', sans-serif;font-size: 15px; color: {{ primary_color }};">
{% trans "Price"|upper %}
......@@ -171,6 +172,7 @@
{% endif %}
</span>
</div>
{% endif %}
</div>
</td>
</tr>
......@@ -202,7 +204,7 @@
<td align="center" style="background-color: black; color: white;font-size: 12px;">
<p style="margin: 10px; font-family: 'Gill Sans', 'Trebuchet MS', sans-serif;">
{% if main_partner %}
<a href="{{ main_partner.link }}"><img src="{% baseurl %}{{ main_partner.logo.url }}" height="70" style="max-width: 170px; height: 70px; margin-bottom: 5px" alt="{{ main_partner.name }}"/></a><br/>
<a href="{{ main_partner.link }}"><img src="{% baseurl %}{% static 'newsletters/sponsor-logo.png' %}" height="70" style="max-width: 170px; height: 70px; margin-bottom: 5px" alt="{{ main_partner.name }}"/></a><br/>
{% trans "our main partner"|upper %}
{% endif %}
</p>
......@@ -212,4 +214,4 @@
</div>
</body>
</html>
{% endwith %}
\ No newline at end of file
{% endwith %}
......@@ -20,7 +20,7 @@
{% trans "What"|upper %}: {{ event.what }}
{% trans "Where"|upper %}: {{ event.where }}
{% trans "When"|upper %}: {% if event.start_datetime == event.end_datetime %}{{ event.start_datetime }}{% elif event.start_datetime|date:'d-M-Y' == event.end_datetime|date:'d-M-Y' %}{{ event.start_datetime }} - {{ event.end_datetime|date:'H:i' }}{% else %}{{ event.start_datetime }} - {{ event.end_datetime }}{% endif %}
{% trans "Price"|upper %}: {% if event.price == 0 %}{% trans "Free" %}{% else %}€{{ event.price }}{% endif %}
{% if event.price is not None %}{% trans "Price"|upper %}: {% if event.price == 0 %}{% trans "Free" %}{% else %}€{{ event.price }}{% endif %}{% endif %}
{% endfor %}
{% trans "room"|capfirst %}: HG00.150
......@@ -29,4 +29,4 @@
{% trans "our main partner"|capfirst %}: {{ main_partner.name }} - {{ main_partner.link }}
{% trans "view this email in your browser"|capfirst %}: {% baseurl %}{{ newsletter.get_absolute_url }}
\ No newline at end of file
{% trans "view this email in your browser"|capfirst %}: {% baseurl %}{{ newsletter.get_absolute_url }}
......@@ -7,4 +7,6 @@ app_name = "newsletters"
urlpatterns = [
url('^(?P<pk>\d+)/$', views.preview, name='preview'),
url(r'admin/send/(?P<pk>\d+)/$', views.admin_send, name='admin-send'),
url('^(?P<year>\d+)/(?P<week>\d+)/nieuwsbrief.html',
views.legacy_redirect, name='legacy-redirect')
]
......@@ -7,6 +7,8 @@ from django.template import Context
from django.template.loader import get_template
from django.utils import translation
from datetime import datetime, timedelta, date
from members.models import Member
from newsletters.models import Newsletter
from partners.models import Partner
......@@ -24,6 +26,17 @@ def preview(request, pk):
})
def legacy_redirect(request, year, week):
newsletter_date = datetime.strptime(
'%s-%s-1' % (year, week), '%Y-%W-%w')
if date(int(year), 1, 4).isoweekday() > 4:
newsletter_date -= timedelta(days=7)
newsletter = get_object_or_404(Newsletter, date=newsletter_date)
return redirect(newsletter.get_absolute_url(), permanent=True)
@staff_member_required
@permission_required('newsletters.change_event')
def admin_send(request, pk):
......
......@@ -57,6 +57,7 @@ urlpatterns = [
url(r'^pizzas/', include('pizzas.urls')),
url(r'^index\.php/events/ical/feed\.ics', DeprecationFeed()),
url(r'^newsletters/', include('newsletters.urls')),
url(r'^nieuwsbrief/', include('newsletters.urls')), # for legacy reasons
url(r'^association$', TemplateView.as_view(
template_name='singlepages/association.html'), name='association'),
url(r'^', include([ # 'association' menu
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment