Merge branch 'dependencies-image' into 'master'

Create dependencies Docker image

See merge request !1488

.gitlab-ci.yml

-variables:
-  # https://hub.docker.com/r/thalia/python-thalia/
-  # https://github.com/thaliawww/python-thalia
-  # Should get auto-updated with the official 'python' repository
-  # Installs:
-  #  - pip: coverage, poetry
-  #  - apt: ghostscript
-  PY38_IMAGE: thalia/python-thalia:3.8
-  PIP_CACHE_DIR: "${CI_PROJECT_DIR}/pip-cache"
+---
 
-stages:
-  - test
-  - deploy
-
-codestyle:
-  stage: test
-  image: $PY38_IMAGE
-  before_script:
-    - poetry install --no-interaction
-  script:
-    - black --check .
-    # Check for obsolete translations in .po files (starting with `#~`).
-    - cd website
-    - grep --include="*.po" --files-with-matches --recursive "^#~" && exit 1 || echo "No obsolete translations found."
-    # Check for untranslated strings in .po files
-    - empty_strings=$(sed '$a\\' **/locale/nl/LC_MESSAGES/django.po | tac | sed '/^$/N;/\nmsgstr ""$/,/^msgid/!d' | tac)
-    - empty_strings+=$(sed '$a\\' locale/nl/LC_MESSAGES/django.po | tac | sed '/^$/N;/\nmsgstr ""$/,/^msgid/!d' | tac)
-    - if [[ $empty_strings ]]; then echo $empty_strings && exit 1; else echo "No untranslated strings found."; fi
-    # Check for fuzzy translations in .po files
-    - grep --include="*.po" --files-with-matches --recursive "#, fuzzy" && exit 1 || echo "No fuzzy translations found."
-
-django-tests:
-  image: $PY38_IMAGE
-  stage: test
-  services:
-    - postgres:latest
-  variables:
-    POSTGRES_DB: thalia
-    POSTGRES_USER: postgres
-    POSTGRES_PASSWORD: ""
-  before_script:
-    - git log -1
-    - poetry install --no-interaction
-  script:
-    - poetry run python website/manage.py check
-    - poetry run python website/manage.py templatecheck --project-only
-    - poetry run python website/manage.py makemigrations --no-input --check --dry-run
-    - poetry run coverage run website/manage.py test website/
-    - coverage report --fail-under=100 --omit website/registrations/urls.py website/registrations/**.py
-    - coverage report --fail-under=100 --omit website/payments/urls.py website/payments/**.py
-    - coverage report
-  after_script:
-    - coverage html --directory=covhtml --title="${CI_COMMIT_REF_SLUG} Coverage Report"
-  artifacts:
-    paths:
-      - covhtml
-
-docs tests:
-  stage: test
-  image: $PY38_IMAGE
-  before_script:
-    # install django deps
-    - poetry install --no-interaction --extras "docs"
-  script:
-    - echo "Building current docs"
-    - cd docs
-    - env -u GITLAB_CI poetry run make doctest
-    - env -u GITLAB_CI poetry run sphinx-build -W . _build
-    - echo "Checking if there are changes"
-    - poetry run ./generate-apidocs.sh
-    - git diff --exit-code
-  artifacts:
-    paths:
-      - docs/_build
-
-build docker image:
-  stage: "test"
-  image: "docker:git"
-  services:
-    - "docker:dind" 
-  tags:
-    - "docker"
-  except:
-    - "tags"
-  before_script:
-    - "echo $DOCKER_REGISTRY_PASSWORD | docker login --username thaliawww --password-stdin registry.hub.docker.com"
-  script:
-    - "docker build --quiet --build-arg install_dev_requirements=${DEV_REQUIREMENTS} --build-arg source_commit=$(git rev-parse HEAD) --tag ${DOCKER_LATEST} ."
-    - "docker tag ${DOCKER_LATEST} ${DOCKER_TAG}"
-    - "docker push ${DOCKER_TAG}"
-  variables:
-    DEV_REQUIREMENTS: 1
-    DOCKER_LATEST: "registry.hub.docker.com/thalia/concrexit:latest"
-    DOCKER_TAG: "registry.hub.docker.com/thalia/concrexit:${CI_COMMIT_SHA}"
-
-build production docker image:
-  extends: "build docker image"
-  only:
-    - "tags"
-  except:
-    - "master"
-  after_script:
-    - "docker tag ${DOCKER_TAG} ${DOCKER_TAG_PRODUCTION}"
-    - "docker tag ${DOCKER_TAG} ${DOCKER_LATEST}"
-    - "docker push ${DOCKER_TAG_PRODUCTION}"
-    - "docker push ${DOCKER_LATEST}"
-  variables:
-    DOCKER_TAG_PRODUCTION: "registry.hub.docker.com/thalia/concrexit:${CI_COMMIT_TAG}"
-    DEV_REQUIREMENTS: 0
-
-coverage deploy:
-  stage: "deploy"
-  image: "python:latest"
-  dependencies:
-    - "django-tests"
-  environment:
-    name: "coverage/${CI_COMMIT_REF_NAME}"
-    url: "https://s3-eu-west-1.amazonaws.com/thalia-coverage/${CI_COMMIT_REF_SLUG}/index.html"
-  before_script:
-      - "pip install awscli"
-  script:
-    - "aws s3 sync --only-show-errors covhtml s3://thalia-coverage/${CI_COMMIT_REF_SLUG}/"
-
-docs deploy:
-  stage: "deploy"
-  image: "python:latest"
-  dependencies:
-    - "docs tests"
-  environment:
-    name: "documentation/${CI_COMMIT_REF_NAME}"
-    url: "https://s3-eu-west-1.amazonaws.com/thalia-documentation/${CI_COMMIT_REF_SLUG}/index.html"
-  before_script:
-      - "pip install awscli"
-  script:
-    - "aws s3 sync --only-show-errors docs/_build s3://thalia-documentation/${CI_COMMIT_REF_SLUG}/"
-
-.reviewsetup:
-  when: "manual"
-  image: "python:latest"
-  variables:
-      AWS_DEFAULT_REGION: "eu-west-1"
-  before_script:
-    - "DEBIAN_FRONTEND=noninteractive apt-get --yes --quiet update"
-    - "DEBIAN_FRONTEND=noninteractive apt-get --yes --quiet install jq"
-    - "pip install awscli"
-
-review create:
-  extends: ".reviewsetup"
-  stage: "deploy"
-  environment:
-    name: "review/${CI_COMMIT_REF_NAME}"
-    url: "https://${CI_COMMIT_REF_SLUG}.public.review.technicie.nl/"
-    on_stop: "review remove"
-  script:
-    - "username=$(head /dev/urandom | tr -dc 'a-z' | head -c 10)"
-    - "password=$(head /dev/urandom | tr -dc 'a-zA-Z' | head -c 32)"
-    - >-
-      sed --in-place
-      --expression "s/@version@/$CI_COMMIT_SHA/g"
-      --expression "s/@username@/$username/g"
-      --expression "s/@password@/$password/g"
-      ./resources/ec2-bootstrap.sh
-    - "resources/review-host-create.sh"
-    - 'echo -e "The deployment is done. Please wait for the website to come up. You can login on https://${CI_COMMIT_REF_SLUG}.public.review.technicie.nl/ with:\nUsername:$username\nPassword:$password"'
-
-
-review remove:
-  extends: ".reviewsetup"
-  stage: "deploy"
-  environment:
-    name: "review/${CI_COMMIT_REF_NAME}"
-    action: "stop"
-  script:
-    - "resources/review-host-remove.sh"
-
-cache:
-  key: "$CI_JOB_NAME"
-  paths:
-    - "${PIP_CACHE_DIR}"
+include:
+  - resources/continuous-integration/config/test.yaml
+  - resources/continuous-integration/config/build.yaml
+  - resources/continuous-integration/config/deploy-static-sites.yaml
+  - resources/continuous-integration/config/deploy-review.yaml

Dockerfile

-FROM python:3.8
+FROM thalia/concrexit-dependencies
 MAINTAINER Thalia Technicie <www@thalia.nu>
 LABEL description="Contains the Thaliawebsite Django application"
 
-# Arguments
 ARG install_dev_requirements=1
 ARG source_commit="unknown"
 
-# Try to keep static operation on top to maximise Docker cache utilisation
-
-# Disable output buffering
 ENV PYTHONUNBUFFERED 1
-ENV DEBIAN_FRONTEND=noninteractive
 ENV SOURCE_COMMIT=${source_commit}
-ENV PATH /root/.poetry/bin:${PATH}
 
-# Set up entrypoint and command
 ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
 
-# Create /concrexit dir
-# Create log dir and log file
-# Create app dir
-RUN mkdir /concrexit && \
-    mkdir -p /concrexit/log/ && \
-    touch /concrexit/log/uwsgi.log && \
-    chown -R www-data:www-data /concrexit && \
-    mkdir -p /usr/src/app && \
-    mkdir -p /usr/src/app/website
-
-# Install dependencies
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    postgresql-client \
-    gettext \
-    ghostscript && \
-    rm -rf /var/lib/apt
+COPY resources/entrypoint.sh resources/entrypoint_production.sh /usr/local/bin/
 
-RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python && \
-    poetry config virtualenvs.create false
-
-WORKDIR /usr/src/app/website/
-# install python requirements
-COPY pyproject.toml /usr/src/app/website/
-COPY poetry.lock /usr/src/app/website/
-RUN if [ "$install_dev_requirements" -eq 1 ]; then \
-        poetry install --no-interaction; \
-    else \
-        echo "This will fail if the dependencies are out of date"; \
-        poetry install --no-interaction --no-dev; \
-    fi; \
-    poetry cache clear --all --no-interaction pypi
-
-# Create entry points
-COPY resources/entrypoint.sh /usr/local/bin/entrypoint.sh
-COPY resources/entrypoint_production.sh /usr/local/bin/entrypoint_production.sh
-RUN chmod +x /usr/local/bin/entrypoint.sh && \
-    chmod +x /usr/local/bin/entrypoint_production.sh
+RUN mkdir --parents /concrexit/log/ && \
+    touch /concrexit/log/uwsgi.log && \
+    chown --recursive www-data:www-data /concrexit/ && \
+    chmod +x /usr/local/bin/entrypoint.sh /usr/local/bin/entrypoint_production.sh
 
-# copy app source
 COPY website /usr/src/app/website/
-
-RUN echo "Don't build releases yourself, let CI do it!"
+WORKDIR /usr/src/app/website/

Dockerfile.dependencies

+FROM python:3.8
+MAINTAINER Thalia Technicie <www@thalia.nu>
+LABEL description="Contains the Thaliawebsite dependencies"
+
+ARG install_dev_requirements=1
+ENV PATH /root/.poetry/bin:${PATH}
+
+WORKDIR /usr/src/app/
+
+COPY pyproject.toml poetry.lock /usr/src/app/
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install --yes --quiet --no-install-recommends \
+        postgresql-client \
+        gettext ghostscript && \
+    rm --recursive --force /var/lib/apt/lists/* && \
+    \
+    curl --silent --show-error --location https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python && \
+    poetry config virtualenvs.create false && \
+    \
+    if [ "$install_dev_requirements" -eq 1 ]; then \
+        poetry install --no-interaction --extras "docs"; \
+    else \
+        poetry install --no-interaction --no-dev; \
+    fi; \
+    poetry cache clear --all --no-interaction pypi

poetry.lock

@@ -216,6 +216,11 @@ optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4"
 version = "5.0.3"
 
+[package.dependencies]
+[package.dependencies.toml]
+optional = true
+version = "*"
+
 [package.extras]
 toml = ["toml"]
 
@@ -1100,7 +1105,7 @@ version = "0.5.1"
 docs = ["recommonmark", "sphinx"]
 
 [metadata]
-content-hash = "33ad4a9027b93aa978de168d98b2b40dc4ad7af5ed4e921f1d214109b3953dab"
+content-hash = "9f5b76b94671370f3f875389e3900868d364017de125a8dbbeb011b3bad48e9d"
 python-versions = "^3.7"
 
 [metadata.files]
@@ -1486,11 +1491,6 @@ markupsafe = [
     {file = "MarkupSafe-1.1.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6"},
     {file = "MarkupSafe-1.1.1-cp37-cp37m-win32.whl", hash = "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2"},
     {file = "MarkupSafe-1.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c"},
-    {file = "MarkupSafe-1.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15"},
-    {file = "MarkupSafe-1.1.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2"},
-    {file = "MarkupSafe-1.1.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42"},
-    {file = "MarkupSafe-1.1.1-cp38-cp38-win32.whl", hash = "sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b"},
-    {file = "MarkupSafe-1.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be"},
     {file = "MarkupSafe-1.1.1.tar.gz", hash = "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b"},
 ]
 msgpack = [

pyproject.toml

@@ -40,7 +40,7 @@ django-template-check = "0.3.1"
 factory_boy = "2.12"
 pydenticon = "0.3.1"
 Faker = "4.0"
-coverage = "5.0.3"
+coverage = {extras = ["toml"], version = "^5.0.3"}
 black = "19.10b0"
 
 [tool.coverage.run]

resources/continuous-integration/config/build.yaml

+.build-docker-image-setup:
+  stage: test
+  tags:
+    - docker
+  image: docker
+  services:
+    - docker:dind
+  before_script:
+    - echo "$DOCKER_REGISTRY_PASSWORD" | docker login --username "thaliawww" --password-stdin "registry.hub.docker.com"
+  variables:
+    DOCKER_LATEST: registry.hub.docker.com/thalia/concrexit:latest
+    DOCKER_TAG: registry.hub.docker.com/thalia/concrexit:${CI_COMMIT_SHA}
+    DOCKER_DEPENDENCIES_LATEST: registry.hub.docker.com/thalia/concrexit-dependencies:latest
+    DOCKER_TAG_PRODUCTION: registry.hub.docker.com/thalia/concrexit:${CI_COMMIT_TAG}
+
+schedule:build-docker-image-dependencies:
+  extends: .build-docker-image-setup
+  only:
+    - schedules
+  script:
+    - docker build --file "Dockerfile.dependencies" --quiet --build-arg "install_dev_requirements=1" --tag "${DOCKER_DEPENDENCIES_LATEST}" .
+    - docker push "${DOCKER_DEPENDENCIES_LATEST}"
+
+build-docker-image:
+  extends: .build-docker-image-setup
+  except:
+    - /v[\d\.]+/
+    - schedules
+  script:
+    - docker build --quiet --build-arg "source_commit=${CI_COMMIT_SHA}" --tag "${DOCKER_LATEST}" .
+    - docker tag "${DOCKER_LATEST}" "${DOCKER_TAG}"
+    - docker push "${DOCKER_TAG}"
+
+build-docker-image-production:
+  extends: .build-docker-image-setup
+  only:
+    - /v[\d\.]+/
+  script:
+    - docker build --file "Dockerfile.dependencies" --quiet --build-arg "install_dev_requirements=0" --tag "${DOCKER_DEPENDENCIES_LATEST}" .
+    - docker build --quiet --build-arg "source_commit=${CI_COMMIT_SHA}" --tag "${DOCKER_LATEST}" .
+    - docker tag "${DOCKER_LATEST}" "${DOCKER_TAG_PRODUCTION}"
+    - docker push "${DOCKER_TAG_PRODUCTION}"
+    - docker push "${DOCKER_LATEST}"

resources/continuous-integration/config/deploy-review.yaml

+---
+
+.review-setup:
+  when: manual
+  stage: deploy
+  except:
+    - schedules
+  image: python:latest
+  before_script:
+    - DEBIAN_FRONTEND=noninteractive apt-get --yes --quiet update
+    - DEBIAN_FRONTEND=noninteractive apt-get --yes --quiet install jq
+    - pip install awscli
+  variables:
+      AWS_DEFAULT_REGION: eu-west-1
+
+review-create:
+  extends: .review-setup
+  environment:
+    name: review/${CI_COMMIT_REF_NAME}
+    url: https://${CI_COMMIT_REF_SLUG}.public.review.technicie.nl/
+    on_stop: review-remove
+  script:
+    - username=$(head /dev/urandom | tr -dc 'a-z' | head -c 10)
+    - password=$(head /dev/urandom | tr -dc 'a-zA-Z' | head -c 32)
+    - >-
+      sed --in-place
+      --expression "s/@version@/$CI_COMMIT_SHA/g"
+      --expression "s/@username@/$username/g"
+      --expression "s/@password@/$password/g"
+      resources/continuous-integration/review/ec2-bootstrap.sh
+    - resources/continuous-integration/review/review-host-create.sh
+    - echo -e "The deployment is done. Please wait for the website to come up. You can login on https://${CI_COMMIT_REF_SLUG}.public.review.technicie.nl/ with:\nUsername:$username\nPassword:$password"
+
+review-remove:
+  extends: .review-setup
+  environment:
+    name: review/${CI_COMMIT_REF_NAME}
+    action: stop
+  script:
+    - resources/continuous-integration/review/

resources/continuous-integration/config/deploy-static-sites.yaml

+---
+
+coverage-deploy:
+  stage: deploy
+  except:
+    - schedules
+  dependencies:
+    - django-tests
+  environment:
+    name: coverage/${CI_COMMIT_REF_NAME}
+    url: https://s3-eu-west-1.amazonaws.com/thalia-coverage/${CI_COMMIT_REF_SLUG}/index.html
+  image: python:latest
+  before_script:
+      - pip install awscli
+  script:
+    - aws s3 sync --only-show-errors "covhtml" "s3://thalia-coverage/${CI_COMMIT_REF_SLUG}/"
+
+docs-deploy:
+  stage: deploy
+  except:
+    - schedules
+  dependencies:
+    - docs-tests
+  environment:
+    name: documentation/${CI_COMMIT_REF_NAME}
+    url: https://s3-eu-west-1.amazonaws.com/thalia-documentation/${CI_COMMIT_REF_SLUG}/index.html
+  image: python:latest
+  before_script:
+      - pip install awscli
+  script:
+    - aws s3 sync --only-show-errors "docs/_build" "s3://thalia-documentation/${CI_COMMIT_REF_SLUG}/"

resources/continuous-integration/config/test.yaml

+---
+
+codestyle:
+  stage: test
+  except:
+    - schedules
+  image: thalia/concrexit-dependencies
+  before_script:
+    - poetry install --no-interaction
+  script:
+    - black --quiet --check website
+    # Check for obsolete translations in .po files (starting with `#~`).
+    - grep --include="*.po" --files-with-matches --recursive "^#~" website && exit 1 || echo "No obsolete translations found."
+    # Check for untranslated strings in .po files
+    - empty_strings=$(sed '$a\\' website/**/locale/nl/LC_MESSAGES/django.po | tac | sed '/^$/N;/\nmsgstr ""$/,/^msgid/!d' | tac)
+    - empty_strings+=$(sed '$a\\' website/locale/nl/LC_MESSAGES/django.po | tac | sed '/^$/N;/\nmsgstr ""$/,/^msgid/!d' | tac)
+    - if [[ $empty_strings ]]; then echo $empty_strings && exit 1; else echo "No untranslated strings found."; fi
+    # Check for fuzzy translations in .po files
+    - grep --include="*.po" --files-with-matches --recursive "#, fuzzy" website && exit 1 || echo "No fuzzy translations found."
+
+django-tests:
+  stage: test
+  except:
+    - schedules
+  image: thalia/concrexit-dependencies
+  services:
+    - postgres:latest
+  before_script:
+    - git log -1
+    - poetry install --no-interaction
+  script:
+    - poetry run python website/manage.py check
+    - poetry run python website/manage.py templatecheck --project-only
+    - poetry run python website/manage.py makemigrations --no-input --check --dry-run
+    - poetry run coverage run website/manage.py test website/
+    - coverage report --fail-under=100 --omit "website/registrations/urls.py" website/registrations/**.py
+    - coverage report --fail-under=100 --omit "website/payments/urls.py" website/payments/**.py
+    - coverage report
+  after_script:
+    - coverage html --directory=covhtml --title="${CI_COMMIT_REF_SLUG} Coverage Report"
+  variables:
+    POSTGRES_DB: thalia
+    POSTGRES_USER: postgres
+    POSTGRES_PASSWORD: ""
+  artifacts:
+    paths:
+      - covhtml
+
+docs-tests:
+  stage: test
+  except:
+    - schedules
+  image: thalia/concrexit-dependencies
+  before_script:
+    - poetry install --no-interaction --extras "docs"
+  script:
+    - echo "Building current docs"
+    - cd docs
+    - env -u GITLAB_CI poetry run make doctest
+    - env -u GITLAB_CI poetry run sphinx-build -W . _build
+    - echo "Checking if there are changes"
+    - poetry run ./generate-apidocs.sh
+    - git diff --exit-code
+  artifacts:
+    paths:
+      - docs/_build

resources/review-host-create.sh → resources/continuous-integration/review/review-host-create.sh

@@ -22,7 +22,7 @@ new_instance_id=$(
     aws ec2 run-instances \
     --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=concrexit-review-${CI_COMMIT_REF_SLUG}}]" \
     --launch-template "LaunchTemplateId=lt-03762fc23450c2471,Version=5" \
-    --user-data file://resources/ec2-bootstrap.sh | 
+    --user-data file://resources/continuous-integration/review/ec2-bootstrap.sh | 
         jq --raw-output ".Instances[0].InstanceId"
     )
 aws ec2 wait instance-running --instance-ids "${new_instance_id}"