Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
thalia
concrexit
Commits
7d111e71
Commit
7d111e71
authored
Nov 23, 2016
by
Sébastiaan Versteeg
Browse files
Finish newsletters migration script and docs
parent
22828ab9
Changes
4
Hide whitespace changes
Inline
Side-by-side
migration-requirements.txt
View file @
7d111e71
requests
lxml
beautifulsoup4
dateparser
migration.md
View file @
7d111e71
...
...
@@ -50,6 +50,10 @@ This migration typically takes a few minutes.
## Newsletters
Make sure that the API key is available as
`MIGRATION_KEY`
in settings.py.
To migrate the newsletters simply execute
`python manage.py migratenewsletters`
.
## Partners
To migrate partners, partner events, vacancy categories and vacancies,
...
...
website/newsletters/management/commands/migratenewsletters.py
View file @
7d111e71
import
requests
import
re
import
dateparser
from
bs4
import
BeautifulSoup
from
datetime
import
datetime
from
datetime
import
datetime
,
timedelta
,
date
from
django.utils
import
timezone
from
django.core.management.base
import
BaseCommand
from
newsletters.models
import
Newsletter
,
NewsletterItem
,
NewsletterEvent
def
isoweek_firstday
(
year
,
week
):
ret
=
datetime
.
strptime
(
'%04d-%02d-1'
%
(
year
,
week
),
'%Y-%W-%w'
)
if
date
(
year
,
1
,
4
).
isoweekday
()
>
4
:
ret
-=
timedelta
(
days
=
7
)
return
ret
class
Command
(
BaseCommand
):
help
=
'Scrapes the newsletters from the old Thalia website'
...
...
@@ -24,7 +33,7 @@ class Command(BaseCommand):
for
year
in
year_range
:
for
week
in
week_range
:
request
=
session
.
get
(
url
.
format
(
year
,
week
))
src
=
request
.
text
src
=
request
.
text
if
request
.
status_code
!=
200
:
continue
...
...
@@ -35,9 +44,7 @@ class Command(BaseCommand):
title
=
"Newsletter week {} {}"
.
format
(
week
,
year
)
newsletter
.
title_nl
=
title
newsletter
.
title_en
=
title
newsletter
.
date
=
datetime
.
strptime
(
'{}-W{}-0'
.
format
(
year
,
week
-
1
),
"%Y-W%W-%w"
).
date
()
newsletter
.
date
=
isoweek_firstday
(
year
,
week
)
all_tr
=
soup
.
find
(
"table"
).
find_all
(
"tr"
)
...
...
@@ -76,21 +83,55 @@ class Command(BaseCommand):
event_data
=
content_td
[
1
].
find_all
(
"span"
)
what
=
event_data
[
0
].
text
where
=
event_data
[
1
].
text
when
=
event_data
[
2
]
when
=
event_data
[
2
].
text
when
=
when
.
replace
(
'vanaf'
,
''
)
parse_settings
=
{
'PREFER_DATES_FROM'
:
'future'
}
if
"-"
in
when
:
when
=
when
.
split
(
"-"
,
2
)
start_when
=
dateparser
.
parse
(
when
[
0
],
settings
=
parse_settings
)
end_when
=
dateparser
.
parse
(
when
[
1
],
settings
=
parse_settings
)
if
len
(
when
[
1
])
<
8
:
end_when
=
end_when
.
replace
(
start_when
.
year
,
start_when
.
month
,
start_when
.
day
)
else
:
start_when
=
end_when
=
dateparser
.
parse
(
when
,
settings
=
parse_settings
)
tz
=
timezone
.
get_current_timezone
()
if
start_when
is
None
and
end_when
is
None
:
start_when
=
end_when
=
newsletter
.
date
elif
start_when
is
None
:
start_when
=
end_when
elif
end_when
is
None
:
end_when
=
start_when
start_when
=
tz
.
localize
(
start_when
)
end_when
=
tz
.
localize
(
end_when
)
start_when
=
start_when
.
replace
(
year
)
end_when
=
end_when
.
replace
(
year
)
item
.
start_datetime
=
start_when
item
.
end_datetime
=
end_when
price
=
None
if
(
"PRIJS"
in
content_td
[
1
].
text
and
event_data
[
3
]
is
not
None
):
price
=
event_data
[
3
].
text
item
.
what_nl
=
what
[:
len
(
what
)
-
1
]
item
.
what_en
=
what
[:
len
(
what
)
-
1
]
item
.
title_nl
=
item
.
what_nl
=
what
[:
len
(
what
)
-
1
]
item
.
title_en
=
item
.
what_en
=
what
[:
len
(
what
)
-
1
]
item
.
where_nl
=
where
[:
len
(
where
)
-
1
]
item
.
where_en
=
where
[:
len
(
where
)
-
1
]
item
.
start_datetime
=
datetime
.
now
()
item
.
end_datetime
=
datetime
.
now
()
if
price
is
not
None
and
"ratis"
not
in
price
:
price
=
price
.
replace
(
","
,
"."
)
price
=
price
.
replace
(
"-"
,
"0"
)
...
...
@@ -112,13 +153,11 @@ class Command(BaseCommand):
.
replace
(
"-"
,
"0"
))
elif
item
.
price
is
not
None
:
item
.
penalty_costs
=
item
.
price
pass
else
:
item
=
NewsletterItem
()
item
.
title_nl
=
title
.
lower
().
capitalize
()
item
.
title_en
=
title
.
lower
().
capitalize
()
item
.
title_nl
=
title
.
lower
().
capitalize
()
item
.
title_en
=
title
.
lower
().
capitalize
()
item
.
description_nl
=
description
item
.
description_en
=
description
item
.
newsletter
=
newsletter
...
...
website/newsletters/templates/newsletters/email.html
View file @
7d111e71
...
...
@@ -148,7 +148,7 @@
</b>
<br>
<span
style=
"font-family: Arial, sans-serif; font-size: 15px; color: white; "
>
{% if event.start_datetime == event.end_datetime %}
{% if event.start_datetime
|date:'d-M-Y H:i'
== event.end_datetime
|date:'d-M-Y H:i'
%}
{{ event.start_datetime }}
{% elif event.start_datetime|date:'d-M-Y' == event.end_datetime|date:'d-M-Y' %}
{{ event.start_datetime }} - {{ event.end_datetime|date:'H:i' }}
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment