Commit 57f7ac93 authored by Joren Vrancken's avatar Joren Vrancken Committed by Sébastiaan Versteeg

Create dependencies Docker image

parent 5dd7acaf
variables:
# https://hub.docker.com/r/thalia/python-thalia/
# https://github.com/thaliawww/python-thalia
# Should get auto-updated with the official 'python' repository
# Installs:
# - pip: coverage, poetry
# - apt: ghostscript
PY38_IMAGE: thalia/python-thalia:3.8
PIP_CACHE_DIR: "${CI_PROJECT_DIR}/pip-cache"
---
stages:
- test
- deploy
codestyle:
stage: test
image: $PY38_IMAGE
before_script:
- poetry install --no-interaction
script:
- black --check .
# Check for obsolete translations in .po files (starting with `#~`).
- cd website
- grep --include="*.po" --files-with-matches --recursive "^#~" && exit 1 || echo "No obsolete translations found."
# Check for untranslated strings in .po files
- empty_strings=$(sed '$a\\' **/locale/nl/LC_MESSAGES/django.po | tac | sed '/^$/N;/\nmsgstr ""$/,/^msgid/!d' | tac)
- empty_strings+=$(sed '$a\\' locale/nl/LC_MESSAGES/django.po | tac | sed '/^$/N;/\nmsgstr ""$/,/^msgid/!d' | tac)
- if [[ $empty_strings ]]; then echo $empty_strings && exit 1; else echo "No untranslated strings found."; fi
# Check for fuzzy translations in .po files
- grep --include="*.po" --files-with-matches --recursive "#, fuzzy" && exit 1 || echo "No fuzzy translations found."
django-tests:
image: $PY38_IMAGE
stage: test
services:
- postgres:latest
variables:
POSTGRES_DB: thalia
POSTGRES_USER: postgres
POSTGRES_PASSWORD: ""
before_script:
- git log -1
- poetry install --no-interaction
script:
- poetry run python website/manage.py check
- poetry run python website/manage.py templatecheck --project-only
- poetry run python website/manage.py makemigrations --no-input --check --dry-run
- poetry run coverage run website/manage.py test website/
- coverage report --fail-under=100 --omit website/registrations/urls.py website/registrations/**.py
- coverage report --fail-under=100 --omit website/payments/urls.py website/payments/**.py
- coverage report
after_script:
- coverage html --directory=covhtml --title="${CI_COMMIT_REF_SLUG} Coverage Report"
artifacts:
paths:
- covhtml
docs tests:
stage: test
image: $PY38_IMAGE
before_script:
# install django deps
- poetry install --no-interaction --extras "docs"
script:
- echo "Building current docs"
- cd docs
- env -u GITLAB_CI poetry run make doctest
- env -u GITLAB_CI poetry run sphinx-build -W . _build
- echo "Checking if there are changes"
- poetry run ./generate-apidocs.sh
- git diff --exit-code
artifacts:
paths:
- docs/_build
build docker image:
stage: "test"
image: "docker:git"
services:
- "docker:dind"
tags:
- "docker"
except:
- "tags"
before_script:
- "echo $DOCKER_REGISTRY_PASSWORD | docker login --username thaliawww --password-stdin registry.hub.docker.com"
script:
- "docker build --quiet --build-arg install_dev_requirements=${DEV_REQUIREMENTS} --build-arg source_commit=$(git rev-parse HEAD) --tag ${DOCKER_LATEST} ."
- "docker tag ${DOCKER_LATEST} ${DOCKER_TAG}"
- "docker push ${DOCKER_TAG}"
variables:
DEV_REQUIREMENTS: 1
DOCKER_LATEST: "registry.hub.docker.com/thalia/concrexit:latest"
DOCKER_TAG: "registry.hub.docker.com/thalia/concrexit:${CI_COMMIT_SHA}"
build production docker image:
extends: "build docker image"
only:
- "tags"
except:
- "master"
after_script:
- "docker tag ${DOCKER_TAG} ${DOCKER_TAG_PRODUCTION}"
- "docker tag ${DOCKER_TAG} ${DOCKER_LATEST}"
- "docker push ${DOCKER_TAG_PRODUCTION}"
- "docker push ${DOCKER_LATEST}"
variables:
DOCKER_TAG_PRODUCTION: "registry.hub.docker.com/thalia/concrexit:${CI_COMMIT_TAG}"
DEV_REQUIREMENTS: 0
coverage deploy:
stage: "deploy"
image: "python:latest"
dependencies:
- "django-tests"
environment:
name: "coverage/${CI_COMMIT_REF_NAME}"
url: "https://s3-eu-west-1.amazonaws.com/thalia-coverage/${CI_COMMIT_REF_SLUG}/index.html"
before_script:
- "pip install awscli"
script:
- "aws s3 sync --only-show-errors covhtml s3://thalia-coverage/${CI_COMMIT_REF_SLUG}/"
docs deploy:
stage: "deploy"
image: "python:latest"
dependencies:
- "docs tests"
environment:
name: "documentation/${CI_COMMIT_REF_NAME}"
url: "https://s3-eu-west-1.amazonaws.com/thalia-documentation/${CI_COMMIT_REF_SLUG}/index.html"
before_script:
- "pip install awscli"
script:
- "aws s3 sync --only-show-errors docs/_build s3://thalia-documentation/${CI_COMMIT_REF_SLUG}/"
.reviewsetup:
when: "manual"
image: "python:latest"
variables:
AWS_DEFAULT_REGION: "eu-west-1"
before_script:
- "DEBIAN_FRONTEND=noninteractive apt-get --yes --quiet update"
- "DEBIAN_FRONTEND=noninteractive apt-get --yes --quiet install jq"
- "pip install awscli"
review create:
extends: ".reviewsetup"
stage: "deploy"
environment:
name: "review/${CI_COMMIT_REF_NAME}"
url: "https://${CI_COMMIT_REF_SLUG}.public.review.technicie.nl/"
on_stop: "review remove"
script:
- "username=$(head /dev/urandom | tr -dc 'a-z' | head -c 10)"
- "password=$(head /dev/urandom | tr -dc 'a-zA-Z' | head -c 32)"
- >-
sed --in-place
--expression "s/@version@/$CI_COMMIT_SHA/g"
--expression "s/@username@/$username/g"
--expression "s/@password@/$password/g"
./resources/ec2-bootstrap.sh
- "resources/review-host-create.sh"
- 'echo -e "The deployment is done. Please wait for the website to come up. You can login on https://${CI_COMMIT_REF_SLUG}.public.review.technicie.nl/ with:\nUsername:$username\nPassword:$password"'
review remove:
extends: ".reviewsetup"
stage: "deploy"
environment:
name: "review/${CI_COMMIT_REF_NAME}"
action: "stop"
script:
- "resources/review-host-remove.sh"
cache:
key: "$CI_JOB_NAME"
paths:
- "${PIP_CACHE_DIR}"
include:
- resources/continuous-integration/config/test.yaml
- resources/continuous-integration/config/build.yaml
- resources/continuous-integration/config/deploy-static-sites.yaml
- resources/continuous-integration/config/deploy-review.yaml
FROM python:3.8
FROM thalia/concrexit-dependencies
MAINTAINER Thalia Technicie <www@thalia.nu>
LABEL description="Contains the Thaliawebsite Django application"
# Arguments
ARG install_dev_requirements=1
ARG source_commit="unknown"
# Try to keep static operation on top to maximise Docker cache utilisation
# Disable output buffering
ENV PYTHONUNBUFFERED 1
ENV DEBIAN_FRONTEND=noninteractive
ENV SOURCE_COMMIT=${source_commit}
ENV PATH /root/.poetry/bin:${PATH}
# Set up entrypoint and command
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
# Create /concrexit dir
# Create log dir and log file
# Create app dir
RUN mkdir /concrexit && \
mkdir -p /concrexit/log/ && \
touch /concrexit/log/uwsgi.log && \
chown -R www-data:www-data /concrexit && \
mkdir -p /usr/src/app && \
mkdir -p /usr/src/app/website
# Install dependencies
RUN apt-get update && apt-get install -y --no-install-recommends \
postgresql-client \
gettext \
ghostscript && \
rm -rf /var/lib/apt
COPY resources/entrypoint.sh resources/entrypoint_production.sh /usr/local/bin/
RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python && \
poetry config virtualenvs.create false
WORKDIR /usr/src/app/website/
# install python requirements
COPY pyproject.toml /usr/src/app/website/
COPY poetry.lock /usr/src/app/website/
RUN if [ "$install_dev_requirements" -eq 1 ]; then \
poetry install --no-interaction; \
else \
echo "This will fail if the dependencies are out of date"; \
poetry install --no-interaction --no-dev; \
fi; \
poetry cache clear --all --no-interaction pypi
# Create entry points
COPY resources/entrypoint.sh /usr/local/bin/entrypoint.sh
COPY resources/entrypoint_production.sh /usr/local/bin/entrypoint_production.sh
RUN chmod +x /usr/local/bin/entrypoint.sh && \
chmod +x /usr/local/bin/entrypoint_production.sh
RUN mkdir --parents /concrexit/log/ && \
touch /concrexit/log/uwsgi.log && \
chown --recursive www-data:www-data /concrexit/ && \
chmod +x /usr/local/bin/entrypoint.sh /usr/local/bin/entrypoint_production.sh
# copy app source
COPY website /usr/src/app/website/
RUN echo "Don't build releases yourself, let CI do it!"
WORKDIR /usr/src/app/website/
FROM python:3.8
MAINTAINER Thalia Technicie <www@thalia.nu>
LABEL description="Contains the Thaliawebsite dependencies"
ARG install_dev_requirements=1
ENV PATH /root/.poetry/bin:${PATH}
WORKDIR /usr/src/app/
COPY pyproject.toml poetry.lock /usr/src/app/
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install --yes --quiet --no-install-recommends \
postgresql-client \
gettext ghostscript && \
rm --recursive --force /var/lib/apt/lists/* && \
\
curl --silent --show-error --location https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python && \
poetry config virtualenvs.create false && \
\
if [ "$install_dev_requirements" -eq 1 ]; then \
poetry install --no-interaction --extras "docs"; \
else \
poetry install --no-interaction --no-dev; \
fi; \
poetry cache clear --all --no-interaction pypi
......@@ -216,6 +216,11 @@ optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4"
version = "5.0.3"
[package.dependencies]
[package.dependencies.toml]
optional = true
version = "*"
[package.extras]
toml = ["toml"]
......@@ -1100,7 +1105,7 @@ version = "0.5.1"
docs = ["recommonmark", "sphinx"]
[metadata]
content-hash = "33ad4a9027b93aa978de168d98b2b40dc4ad7af5ed4e921f1d214109b3953dab"
content-hash = "9f5b76b94671370f3f875389e3900868d364017de125a8dbbeb011b3bad48e9d"
python-versions = "^3.7"
[metadata.files]
......@@ -1486,11 +1491,6 @@ markupsafe = [
{file = "MarkupSafe-1.1.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6"},
{file = "MarkupSafe-1.1.1-cp37-cp37m-win32.whl", hash = "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2"},
{file = "MarkupSafe-1.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c"},
{file = "MarkupSafe-1.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15"},
{file = "MarkupSafe-1.1.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2"},
{file = "MarkupSafe-1.1.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42"},
{file = "MarkupSafe-1.1.1-cp38-cp38-win32.whl", hash = "sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b"},
{file = "MarkupSafe-1.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be"},
{file = "MarkupSafe-1.1.1.tar.gz", hash = "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b"},
]
msgpack = [
......
......@@ -40,7 +40,7 @@ django-template-check = "0.3.1"
factory_boy = "2.12"
pydenticon = "0.3.1"
Faker = "4.0"
coverage = "5.0.3"
coverage = {extras = ["toml"], version = "^5.0.3"}
black = "19.10b0"
[tool.coverage.run]
......
.build-docker-image-setup:
stage: test
tags:
- docker
image: docker
services:
- docker:dind
before_script:
- echo "$DOCKER_REGISTRY_PASSWORD" | docker login --username "thaliawww" --password-stdin "registry.hub.docker.com"
variables:
DOCKER_LATEST: registry.hub.docker.com/thalia/concrexit:latest
DOCKER_TAG: registry.hub.docker.com/thalia/concrexit:${CI_COMMIT_SHA}
DOCKER_DEPENDENCIES_LATEST: registry.hub.docker.com/thalia/concrexit-dependencies:latest
DOCKER_TAG_PRODUCTION: registry.hub.docker.com/thalia/concrexit:${CI_COMMIT_TAG}
schedule:build-docker-image-dependencies:
extends: .build-docker-image-setup
only:
- schedules
script:
- docker build --file "Dockerfile.dependencies" --quiet --build-arg "install_dev_requirements=1" --tag "${DOCKER_DEPENDENCIES_LATEST}" .
- docker push "${DOCKER_DEPENDENCIES_LATEST}"
build-docker-image:
extends: .build-docker-image-setup
except:
- /v[\d\.]+/
- schedules
script:
- docker build --quiet --build-arg "source_commit=${CI_COMMIT_SHA}" --tag "${DOCKER_LATEST}" .
- docker tag "${DOCKER_LATEST}" "${DOCKER_TAG}"
- docker push "${DOCKER_TAG}"
build-docker-image-production:
extends: .build-docker-image-setup
only:
- /v[\d\.]+/
script:
- docker build --file "Dockerfile.dependencies" --quiet --build-arg "install_dev_requirements=0" --tag "${DOCKER_DEPENDENCIES_LATEST}" .
- docker build --quiet --build-arg "source_commit=${CI_COMMIT_SHA}" --tag "${DOCKER_LATEST}" .
- docker tag "${DOCKER_LATEST}" "${DOCKER_TAG_PRODUCTION}"
- docker push "${DOCKER_TAG_PRODUCTION}"
- docker push "${DOCKER_LATEST}"
---
.review-setup:
when: manual
stage: deploy
except:
- schedules
image: python:latest
before_script:
- DEBIAN_FRONTEND=noninteractive apt-get --yes --quiet update
- DEBIAN_FRONTEND=noninteractive apt-get --yes --quiet install jq
- pip install awscli
variables:
AWS_DEFAULT_REGION: eu-west-1
review-create:
extends: .review-setup
environment:
name: review/${CI_COMMIT_REF_NAME}
url: https://${CI_COMMIT_REF_SLUG}.public.review.technicie.nl/
on_stop: review-remove
script:
- username=$(head /dev/urandom | tr -dc 'a-z' | head -c 10)
- password=$(head /dev/urandom | tr -dc 'a-zA-Z' | head -c 32)
- >-
sed --in-place
--expression "s/@version@/$CI_COMMIT_SHA/g"
--expression "s/@username@/$username/g"
--expression "s/@password@/$password/g"
resources/continuous-integration/review/ec2-bootstrap.sh
- resources/continuous-integration/review/review-host-create.sh
- echo -e "The deployment is done. Please wait for the website to come up. You can login on https://${CI_COMMIT_REF_SLUG}.public.review.technicie.nl/ with:\nUsername:$username\nPassword:$password"
review-remove:
extends: .review-setup
environment:
name: review/${CI_COMMIT_REF_NAME}
action: stop
script:
- resources/continuous-integration/review/
---
coverage-deploy:
stage: deploy
except:
- schedules
dependencies:
- django-tests
environment:
name: coverage/${CI_COMMIT_REF_NAME}
url: https://s3-eu-west-1.amazonaws.com/thalia-coverage/${CI_COMMIT_REF_SLUG}/index.html
image: python:latest
before_script:
- pip install awscli
script:
- aws s3 sync --only-show-errors "covhtml" "s3://thalia-coverage/${CI_COMMIT_REF_SLUG}/"
docs-deploy:
stage: deploy
except:
- schedules
dependencies:
- docs-tests
environment:
name: documentation/${CI_COMMIT_REF_NAME}
url: https://s3-eu-west-1.amazonaws.com/thalia-documentation/${CI_COMMIT_REF_SLUG}/index.html
image: python:latest
before_script:
- pip install awscli
script:
- aws s3 sync --only-show-errors "docs/_build" "s3://thalia-documentation/${CI_COMMIT_REF_SLUG}/"
---
codestyle:
stage: test
except:
- schedules
image: thalia/concrexit-dependencies
before_script:
- poetry install --no-interaction
script:
- black --quiet --check website
# Check for obsolete translations in .po files (starting with `#~`).
- grep --include="*.po" --files-with-matches --recursive "^#~" website && exit 1 || echo "No obsolete translations found."
# Check for untranslated strings in .po files
- empty_strings=$(sed '$a\\' website/**/locale/nl/LC_MESSAGES/django.po | tac | sed '/^$/N;/\nmsgstr ""$/,/^msgid/!d' | tac)
- empty_strings+=$(sed '$a\\' website/locale/nl/LC_MESSAGES/django.po | tac | sed '/^$/N;/\nmsgstr ""$/,/^msgid/!d' | tac)
- if [[ $empty_strings ]]; then echo $empty_strings && exit 1; else echo "No untranslated strings found."; fi
# Check for fuzzy translations in .po files
- grep --include="*.po" --files-with-matches --recursive "#, fuzzy" website && exit 1 || echo "No fuzzy translations found."
django-tests:
stage: test
except:
- schedules
image: thalia/concrexit-dependencies
services:
- postgres:latest
before_script:
- git log -1
- poetry install --no-interaction
script:
- poetry run python website/manage.py check
- poetry run python website/manage.py templatecheck --project-only
- poetry run python website/manage.py makemigrations --no-input --check --dry-run
- poetry run coverage run website/manage.py test website/
- coverage report --fail-under=100 --omit "website/registrations/urls.py" website/registrations/**.py
- coverage report --fail-under=100 --omit "website/payments/urls.py" website/payments/**.py
- coverage report
after_script:
- coverage html --directory=covhtml --title="${CI_COMMIT_REF_SLUG} Coverage Report"
variables:
POSTGRES_DB: thalia
POSTGRES_USER: postgres
POSTGRES_PASSWORD: ""
artifacts:
paths:
- covhtml
docs-tests:
stage: test
except:
- schedules
image: thalia/concrexit-dependencies
before_script:
- poetry install --no-interaction --extras "docs"
script:
- echo "Building current docs"
- cd docs
- env -u GITLAB_CI poetry run make doctest
- env -u GITLAB_CI poetry run sphinx-build -W . _build
- echo "Checking if there are changes"
- poetry run ./generate-apidocs.sh
- git diff --exit-code
artifacts:
paths:
- docs/_build
......@@ -22,7 +22,7 @@ new_instance_id=$(
aws ec2 run-instances \
--tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=concrexit-review-${CI_COMMIT_REF_SLUG}}]" \
--launch-template "LaunchTemplateId=lt-03762fc23450c2471,Version=5" \
--user-data file://resources/ec2-bootstrap.sh |
--user-data file://resources/continuous-integration/review/ec2-bootstrap.sh |
jq --raw-output ".Instances[0].InstanceId"
)
aws ec2 wait instance-running --instance-ids "${new_instance_id}"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment