Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
thalia
concrexit
Commits
57f7ac93
Commit
57f7ac93
authored
Feb 08, 2020
by
Joren Vrancken
Committed by
Sébastiaan Versteeg
Feb 08, 2020
Browse files
Create dependencies Docker image
parent
5dd7acaf
Changes
12
Hide whitespace changes
Inline
Side-by-side
.gitlab-ci.yml
View file @
57f7ac93
variables
:
# https://hub.docker.com/r/thalia/python-thalia/
# https://github.com/thaliawww/python-thalia
# Should get auto-updated with the official 'python' repository
# Installs:
# - pip: coverage, poetry
# - apt: ghostscript
PY38_IMAGE
:
thalia/python-thalia:3.8
PIP_CACHE_DIR
:
"
${CI_PROJECT_DIR}/pip-cache"
---
stages
:
-
test
-
deploy
codestyle
:
stage
:
test
image
:
$PY38_IMAGE
before_script
:
-
poetry install --no-interaction
script
:
-
black --check .
# Check for obsolete translations in .po files (starting with `#~`).
-
cd website
-
grep --include="*.po" --files-with-matches --recursive "^#~" && exit 1 || echo "No obsolete translations found."
# Check for untranslated strings in .po files
-
empty_strings=$(sed '$a\\' **/locale/nl/LC_MESSAGES/django.po | tac | sed '/^$/N;/\nmsgstr ""$/,/^msgid/!d' | tac)
-
empty_strings+=$(sed '$a\\' locale/nl/LC_MESSAGES/django.po | tac | sed '/^$/N;/\nmsgstr ""$/,/^msgid/!d' | tac)
-
if [[ $empty_strings ]]; then echo $empty_strings && exit 1; else echo "No untranslated strings found."; fi
# Check for fuzzy translations in .po files
-
grep --include="*.po" --files-with-matches --recursive "#, fuzzy" && exit 1 || echo "No fuzzy translations found."
django-tests
:
image
:
$PY38_IMAGE
stage
:
test
services
:
-
postgres:latest
variables
:
POSTGRES_DB
:
thalia
POSTGRES_USER
:
postgres
POSTGRES_PASSWORD
:
"
"
before_script
:
-
git log -1
-
poetry install --no-interaction
script
:
-
poetry run python website/manage.py check
-
poetry run python website/manage.py templatecheck --project-only
-
poetry run python website/manage.py makemigrations --no-input --check --dry-run
-
poetry run coverage run website/manage.py test website/
-
coverage report --fail-under=100 --omit website/registrations/urls.py website/registrations/**.py
-
coverage report --fail-under=100 --omit website/payments/urls.py website/payments/**.py
-
coverage report
after_script
:
-
coverage html --directory=covhtml --title="${CI_COMMIT_REF_SLUG} Coverage Report"
artifacts
:
paths
:
-
covhtml
docs tests
:
stage
:
test
image
:
$PY38_IMAGE
before_script
:
# install django deps
-
poetry install --no-interaction --extras "docs"
script
:
-
echo "Building current docs"
-
cd docs
-
env -u GITLAB_CI poetry run make doctest
-
env -u GITLAB_CI poetry run sphinx-build -W . _build
-
echo "Checking if there are changes"
-
poetry run ./generate-apidocs.sh
-
git diff --exit-code
artifacts
:
paths
:
-
docs/_build
build docker image
:
stage
:
"
test"
image
:
"
docker:git"
services
:
-
"
docker:dind"
tags
:
-
"
docker"
except
:
-
"
tags"
before_script
:
-
"
echo
$DOCKER_REGISTRY_PASSWORD
|
docker
login
--username
thaliawww
--password-stdin
registry.hub.docker.com"
script
:
-
"
docker
build
--quiet
--build-arg
install_dev_requirements=${DEV_REQUIREMENTS}
--build-arg
source_commit=$(git
rev-parse
HEAD)
--tag
${DOCKER_LATEST}
."
-
"
docker
tag
${DOCKER_LATEST}
${DOCKER_TAG}"
-
"
docker
push
${DOCKER_TAG}"
variables
:
DEV_REQUIREMENTS
:
1
DOCKER_LATEST
:
"
registry.hub.docker.com/thalia/concrexit:latest"
DOCKER_TAG
:
"
registry.hub.docker.com/thalia/concrexit:${CI_COMMIT_SHA}"
build production docker image
:
extends
:
"
build
docker
image"
only
:
-
"
tags"
except
:
-
"
master"
after_script
:
-
"
docker
tag
${DOCKER_TAG}
${DOCKER_TAG_PRODUCTION}"
-
"
docker
tag
${DOCKER_TAG}
${DOCKER_LATEST}"
-
"
docker
push
${DOCKER_TAG_PRODUCTION}"
-
"
docker
push
${DOCKER_LATEST}"
variables
:
DOCKER_TAG_PRODUCTION
:
"
registry.hub.docker.com/thalia/concrexit:${CI_COMMIT_TAG}"
DEV_REQUIREMENTS
:
0
coverage deploy
:
stage
:
"
deploy"
image
:
"
python:latest"
dependencies
:
-
"
django-tests"
environment
:
name
:
"
coverage/${CI_COMMIT_REF_NAME}"
url
:
"
https://s3-eu-west-1.amazonaws.com/thalia-coverage/${CI_COMMIT_REF_SLUG}/index.html"
before_script
:
-
"
pip
install
awscli"
script
:
-
"
aws
s3
sync
--only-show-errors
covhtml
s3://thalia-coverage/${CI_COMMIT_REF_SLUG}/"
docs deploy
:
stage
:
"
deploy"
image
:
"
python:latest"
dependencies
:
-
"
docs
tests"
environment
:
name
:
"
documentation/${CI_COMMIT_REF_NAME}"
url
:
"
https://s3-eu-west-1.amazonaws.com/thalia-documentation/${CI_COMMIT_REF_SLUG}/index.html"
before_script
:
-
"
pip
install
awscli"
script
:
-
"
aws
s3
sync
--only-show-errors
docs/_build
s3://thalia-documentation/${CI_COMMIT_REF_SLUG}/"
.reviewsetup
:
when
:
"
manual"
image
:
"
python:latest"
variables
:
AWS_DEFAULT_REGION
:
"
eu-west-1"
before_script
:
-
"
DEBIAN_FRONTEND=noninteractive
apt-get
--yes
--quiet
update"
-
"
DEBIAN_FRONTEND=noninteractive
apt-get
--yes
--quiet
install
jq"
-
"
pip
install
awscli"
review create
:
extends
:
"
.reviewsetup"
stage
:
"
deploy"
environment
:
name
:
"
review/${CI_COMMIT_REF_NAME}"
url
:
"
https://${CI_COMMIT_REF_SLUG}.public.review.technicie.nl/"
on_stop
:
"
review
remove"
script
:
-
"
username=$(head
/dev/urandom
|
tr
-dc
'a-z'
|
head
-c
10)"
-
"
password=$(head
/dev/urandom
|
tr
-dc
'a-zA-Z'
|
head
-c
32)"
-
>-
sed --in-place
--expression "s/@version@/$CI_COMMIT_SHA/g"
--expression "s/@username@/$username/g"
--expression "s/@password@/$password/g"
./resources/ec2-bootstrap.sh
-
"
resources/review-host-create.sh"
-
'
echo
-e
"The
deployment
is
done.
Please
wait
for
the
website
to
come
up.
You
can
login
on
https://${CI_COMMIT_REF_SLUG}.public.review.technicie.nl/
with:\nUsername:$username\nPassword:$password"'
review remove
:
extends
:
"
.reviewsetup"
stage
:
"
deploy"
environment
:
name
:
"
review/${CI_COMMIT_REF_NAME}"
action
:
"
stop"
script
:
-
"
resources/review-host-remove.sh"
cache
:
key
:
"
$CI_JOB_NAME"
paths
:
-
"
${PIP_CACHE_DIR}"
include
:
-
resources/continuous-integration/config/test.yaml
-
resources/continuous-integration/config/build.yaml
-
resources/continuous-integration/config/deploy-static-sites.yaml
-
resources/continuous-integration/config/deploy-review.yaml
Dockerfile
View file @
57f7ac93
FROM
python:3.8
FROM
thalia/concrexit-dependencies
MAINTAINER
Thalia Technicie <www@thalia.nu>
LABEL
description="Contains the Thaliawebsite Django application"
# Arguments
ARG
install_dev_requirements=1
ARG
source_commit="unknown"
# Try to keep static operation on top to maximise Docker cache utilisation
# Disable output buffering
ENV
PYTHONUNBUFFERED 1
ENV
DEBIAN_FRONTEND=noninteractive
ENV
SOURCE_COMMIT=${source_commit}
ENV
PATH /root/.poetry/bin:${PATH}
# Set up entrypoint and command
ENTRYPOINT
["/usr/local/bin/entrypoint.sh"]
# Create /concrexit dir
# Create log dir and log file
# Create app dir
RUN
mkdir
/concrexit
&&
\
mkdir
-p
/concrexit/log/
&&
\
touch
/concrexit/log/uwsgi.log
&&
\
chown
-R
www-data:www-data /concrexit
&&
\
mkdir
-p
/usr/src/app
&&
\
mkdir
-p
/usr/src/app/website
# Install dependencies
RUN
apt-get update
&&
apt-get
install
-y
--no-install-recommends
\
postgresql-client
\
gettext
\
ghostscript
&&
\
rm
-rf
/var/lib/apt
COPY
resources/entrypoint.sh resources/entrypoint_production.sh /usr/local/bin/
RUN
curl
-sSL
https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python
&&
\
poetry config virtualenvs.create
false
WORKDIR
/usr/src/app/website/
# install python requirements
COPY
pyproject.toml /usr/src/app/website/
COPY
poetry.lock /usr/src/app/website/
RUN if
[
"
$install_dev_requirements
"
-eq
1
]
;
then
\
poetry
install
--no-interaction
;
\
else
\
echo
"This will fail if the dependencies are out of date"
;
\
poetry
install
--no-interaction
--no-dev
;
\
fi
;
\
poetry cache clear
--all
--no-interaction
pypi
# Create entry points
COPY
resources/entrypoint.sh /usr/local/bin/entrypoint.sh
COPY
resources/entrypoint_production.sh /usr/local/bin/entrypoint_production.sh
RUN
chmod
+x /usr/local/bin/entrypoint.sh
&&
\
chmod
+x /usr/local/bin/entrypoint_production.sh
RUN
mkdir
--parents
/concrexit/log/
&&
\
touch
/concrexit/log/uwsgi.log
&&
\
chown
--recursive
www-data:www-data /concrexit/
&&
\
chmod
+x /usr/local/bin/entrypoint.sh /usr/local/bin/entrypoint_production.sh
# copy app source
COPY
website /usr/src/app/website/
RUN
echo
"Don't build releases yourself, let CI do it!"
WORKDIR
/usr/src/app/website/
Dockerfile.dependencies
0 → 100644
View file @
57f7ac93
FROM python:3.8
MAINTAINER Thalia Technicie <www@thalia.nu>
LABEL description="Contains the Thaliawebsite dependencies"
ARG install_dev_requirements=1
ENV PATH /root/.poetry/bin:${PATH}
WORKDIR /usr/src/app/
COPY pyproject.toml poetry.lock /usr/src/app/
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install --yes --quiet --no-install-recommends \
postgresql-client \
gettext ghostscript && \
rm --recursive --force /var/lib/apt/lists/* && \
\
curl --silent --show-error --location https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python && \
poetry config virtualenvs.create false && \
\
if [ "$install_dev_requirements" -eq 1 ]; then \
poetry install --no-interaction --extras "docs"; \
else \
poetry install --no-interaction --no-dev; \
fi; \
poetry cache clear --all --no-interaction pypi
poetry.lock
View file @
57f7ac93
...
...
@@ -216,6 +216,11 @@ optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4"
version = "5.0.3"
[package.dependencies]
[package.dependencies.toml]
optional = true
version = "*"
[package.extras]
toml = ["toml"]
...
...
@@ -1100,7 +1105,7 @@ version = "0.5.1"
docs = ["recommonmark", "sphinx"]
[metadata]
content-hash = "
33ad4a9027b93aa978de168d98b2b40dc4ad7af5ed4e921f1d214109b3953dab
"
content-hash = "
9f5b76b94671370f3f875389e3900868d364017de125a8dbbeb011b3bad48e9d
"
python-versions = "^3.7"
[metadata.files]
...
...
@@ -1486,11 +1491,6 @@ markupsafe = [
{file = "MarkupSafe-1.1.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6"},
{file = "MarkupSafe-1.1.1-cp37-cp37m-win32.whl", hash = "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2"},
{file = "MarkupSafe-1.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c"},
{file = "MarkupSafe-1.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15"},
{file = "MarkupSafe-1.1.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2"},
{file = "MarkupSafe-1.1.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42"},
{file = "MarkupSafe-1.1.1-cp38-cp38-win32.whl", hash = "sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b"},
{file = "MarkupSafe-1.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be"},
{file = "MarkupSafe-1.1.1.tar.gz", hash = "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b"},
]
msgpack = [
...
...
pyproject.toml
View file @
57f7ac93
...
...
@@ -40,7 +40,7 @@ django-template-check = "0.3.1"
factory_boy
=
"2.12"
pydenticon
=
"0.3.1"
Faker
=
"4.0"
coverage
=
"
5.0.3"
coverage
=
{
extras
=
["toml"]
,
version
=
"^
5.0.3"
}
black
=
"19.10b0"
[tool.coverage.run]
...
...
resources/continuous-integration/config/build.yaml
0 → 100644
View file @
57f7ac93
.build-docker-image-setup
:
stage
:
test
tags
:
-
docker
image
:
docker
services
:
-
docker:dind
before_script
:
-
echo "$DOCKER_REGISTRY_PASSWORD" | docker login --username "thaliawww" --password-stdin "registry.hub.docker.com"
variables
:
DOCKER_LATEST
:
registry.hub.docker.com/thalia/concrexit:latest
DOCKER_TAG
:
registry.hub.docker.com/thalia/concrexit:${CI_COMMIT_SHA}
DOCKER_DEPENDENCIES_LATEST
:
registry.hub.docker.com/thalia/concrexit-dependencies:latest
DOCKER_TAG_PRODUCTION
:
registry.hub.docker.com/thalia/concrexit:${CI_COMMIT_TAG}
schedule:build-docker-image-dependencies:
extends
:
.build-docker-image-setup
only
:
-
schedules
script
:
-
docker build --file "Dockerfile.dependencies" --quiet --build-arg "install_dev_requirements=1" --tag "${DOCKER_DEPENDENCIES_LATEST}" .
-
docker push "${DOCKER_DEPENDENCIES_LATEST}"
build-docker-image
:
extends
:
.build-docker-image-setup
except
:
-
/v[\d\.]+/
-
schedules
script
:
-
docker build --quiet --build-arg "source_commit=${CI_COMMIT_SHA}" --tag "${DOCKER_LATEST}" .
-
docker tag "${DOCKER_LATEST}" "${DOCKER_TAG}"
-
docker push "${DOCKER_TAG}"
build-docker-image-production
:
extends
:
.build-docker-image-setup
only
:
-
/v[\d\.]+/
script
:
-
docker build --file "Dockerfile.dependencies" --quiet --build-arg "install_dev_requirements=0" --tag "${DOCKER_DEPENDENCIES_LATEST}" .
-
docker build --quiet --build-arg "source_commit=${CI_COMMIT_SHA}" --tag "${DOCKER_LATEST}" .
-
docker tag "${DOCKER_LATEST}" "${DOCKER_TAG_PRODUCTION}"
-
docker push "${DOCKER_TAG_PRODUCTION}"
-
docker push "${DOCKER_LATEST}"
resources/continuous-integration/config/deploy-review.yaml
0 → 100644
View file @
57f7ac93
---
.review-setup
:
when
:
manual
stage
:
deploy
except
:
-
schedules
image
:
python:latest
before_script
:
-
DEBIAN_FRONTEND=noninteractive apt-get --yes --quiet update
-
DEBIAN_FRONTEND=noninteractive apt-get --yes --quiet install jq
-
pip install awscli
variables
:
AWS_DEFAULT_REGION
:
eu-west-1
review-create
:
extends
:
.review-setup
environment
:
name
:
review/${CI_COMMIT_REF_NAME}
url
:
https://${CI_COMMIT_REF_SLUG}.public.review.technicie.nl/
on_stop
:
review-remove
script
:
-
username=$(head /dev/urandom | tr -dc 'a-z' | head -c 10)
-
password=$(head /dev/urandom | tr -dc 'a-zA-Z' | head -c 32)
-
>-
sed --in-place
--expression "s/@version@/$CI_COMMIT_SHA/g"
--expression "s/@username@/$username/g"
--expression "s/@password@/$password/g"
resources/continuous-integration/review/ec2-bootstrap.sh
-
resources/continuous-integration/review/review-host-create.sh
-
echo -e "The deployment is done. Please wait for the website to come up. You can login on https://${CI_COMMIT_REF_SLUG}.public.review.technicie.nl/ with:\nUsername:$username\nPassword:$password"
review-remove
:
extends
:
.review-setup
environment
:
name
:
review/${CI_COMMIT_REF_NAME}
action
:
stop
script
:
-
resources/continuous-integration/review/
resources/continuous-integration/config/deploy-static-sites.yaml
0 → 100644
View file @
57f7ac93
---
coverage-deploy
:
stage
:
deploy
except
:
-
schedules
dependencies
:
-
django-tests
environment
:
name
:
coverage/${CI_COMMIT_REF_NAME}
url
:
https://s3-eu-west-1.amazonaws.com/thalia-coverage/${CI_COMMIT_REF_SLUG}/index.html
image
:
python:latest
before_script
:
-
pip install awscli
script
:
-
aws s3 sync --only-show-errors "covhtml" "s3://thalia-coverage/${CI_COMMIT_REF_SLUG}/"
docs-deploy
:
stage
:
deploy
except
:
-
schedules
dependencies
:
-
docs-tests
environment
:
name
:
documentation/${CI_COMMIT_REF_NAME}
url
:
https://s3-eu-west-1.amazonaws.com/thalia-documentation/${CI_COMMIT_REF_SLUG}/index.html
image
:
python:latest
before_script
:
-
pip install awscli
script
:
-
aws s3 sync --only-show-errors "docs/_build" "s3://thalia-documentation/${CI_COMMIT_REF_SLUG}/"
resources/continuous-integration/config/test.yaml
0 → 100644
View file @
57f7ac93
---
codestyle
:
stage
:
test
except
:
-
schedules
image
:
thalia/concrexit-dependencies
before_script
:
-
poetry install --no-interaction
script
:
-
black --quiet --check website
# Check for obsolete translations in .po files (starting with `#~`).
-
grep --include="*.po" --files-with-matches --recursive "^#~" website && exit 1 || echo "No obsolete translations found."
# Check for untranslated strings in .po files
-
empty_strings=$(sed '$a\\' website/**/locale/nl/LC_MESSAGES/django.po | tac | sed '/^$/N;/\nmsgstr ""$/,/^msgid/!d' | tac)
-
empty_strings+=$(sed '$a\\' website/locale/nl/LC_MESSAGES/django.po | tac | sed '/^$/N;/\nmsgstr ""$/,/^msgid/!d' | tac)
-
if [[ $empty_strings ]]; then echo $empty_strings && exit 1; else echo "No untranslated strings found."; fi
# Check for fuzzy translations in .po files
-
grep --include="*.po" --files-with-matches --recursive "#, fuzzy" website && exit 1 || echo "No fuzzy translations found."
django-tests
:
stage
:
test
except
:
-
schedules
image
:
thalia/concrexit-dependencies
services
:
-
postgres:latest
before_script
:
-
git log -1
-
poetry install --no-interaction
script
:
-
poetry run python website/manage.py check
-
poetry run python website/manage.py templatecheck --project-only
-
poetry run python website/manage.py makemigrations --no-input --check --dry-run
-
poetry run coverage run website/manage.py test website/
-
coverage report --fail-under=100 --omit "website/registrations/urls.py" website/registrations/**.py
-
coverage report --fail-under=100 --omit "website/payments/urls.py" website/payments/**.py
-
coverage report
after_script
:
-
coverage html --directory=covhtml --title="${CI_COMMIT_REF_SLUG} Coverage Report"
variables
:
POSTGRES_DB
:
thalia
POSTGRES_USER
:
postgres
POSTGRES_PASSWORD
:
"
"
artifacts
:
paths
:
-
covhtml
docs-tests
:
stage
:
test
except
:
-
schedules
image
:
thalia/concrexit-dependencies
before_script
:
-
poetry install --no-interaction --extras "docs"
script
:
-
echo "Building current docs"
-
cd docs
-
env -u GITLAB_CI poetry run make doctest
-
env -u GITLAB_CI poetry run sphinx-build -W . _build
-
echo "Checking if there are changes"
-
poetry run ./generate-apidocs.sh
-
git diff --exit-code
artifacts
:
paths
:
-
docs/_build
resources/ec2-bootstrap.sh
→
resources/
continuous-integration/review/
ec2-bootstrap.sh
View file @
57f7ac93
File moved
resources/review-host-create.sh
→
resources/
continuous-integration/review/
review-host-create.sh
View file @
57f7ac93
...
...
@@ -22,7 +22,7 @@ new_instance_id=$(
aws ec2 run-instances
\
--tag-specifications
"ResourceType=instance,Tags=[{Key=Name,Value=concrexit-review-
${
CI_COMMIT_REF_SLUG
}
}]"
\
--launch-template
"LaunchTemplateId=lt-03762fc23450c2471,Version=5"
\
--user-data
file://resources/ec2-bootstrap.sh |
--user-data
file://resources/
continuous-integration/review/
ec2-bootstrap.sh |
jq
--raw-output
".Instances[0].InstanceId"
)
aws ec2
wait
instance-running
--instance-ids
"
${
new_instance_id
}
"
...
...
resources/review-host-remove.sh
→
resources/
continuous-integration/review/
review-host-remove.sh
View file @
57f7ac93
File moved
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment