.gitlab-ci.yml 7.46 KB
Newer Older
Wietse Kuipers's avatar
Wietse Kuipers committed
1 2 3 4
variables:
  POSTGRES_DB: thalia
  POSTGRES_USER: postgres
  POSTGRES_PASSWORD: ""
5 6
  # https://hub.docker.com/r/thalia/python-thalia/
  # https://github.com/thaliawww/python-thalia
7 8
  # Should get auto-updated with the official 'python' repository
  # Installs:
Jelle Besseling's avatar
Jelle Besseling committed
9
  #  - pip: coverage, poetry
10
  #  - apt: ghostscript
Sébastiaan Versteeg's avatar
Sébastiaan Versteeg committed
11
  PY38_IMAGE: thalia/python-thalia:3.8
Thom Wiggers's avatar
Thom Wiggers committed
12
  PIP_CACHE_DIR: "${CI_PROJECT_DIR}/pip-cache"
Wietse Kuipers's avatar
Wietse Kuipers committed
13

Thom Wiggers's avatar
Thom Wiggers committed
14 15 16 17
stages:
  - test
  - deploy

18
codestyle:
Thom Wiggers's avatar
Thom Wiggers committed
19
  stage: test
Joren Vrancken's avatar
Joren Vrancken committed
20
  image: $PY38_IMAGE
Jelle Besseling's avatar
Jelle Besseling committed
21 22
  before_script:
    - poetry install --no-interaction
Thom Wiggers's avatar
Thom Wiggers committed
23
  script:
Luko van der Maas's avatar
Luko van der Maas committed
24
    - black --check .
25
    # Check for obsolete translations in .po files (starting with `#~`).
26
    - cd website
27
    - grep --include="*.po" --files-with-matches --recursive "^#~" && exit 1 || echo "No obsolete translations found."
28 29 30 31 32 33
    # Check for untranslated strings in .po files
    - empty_strings=$(sed '$a\\' **/locale/nl/LC_MESSAGES/django.po | tac | sed '/^$/N;/\nmsgstr ""$/,/^msgid/!d' | tac)
    - empty_strings+=$(sed '$a\\' locale/nl/LC_MESSAGES/django.po | tac | sed '/^$/N;/\nmsgstr ""$/,/^msgid/!d' | tac)
    - if [[ $empty_strings ]]; then echo $empty_strings && exit 1; else echo "No untranslated strings found."; fi
    # Check for fuzzy translations in .po files
    - grep --include="*.po" --files-with-matches --recursive "#, fuzzy" && exit 1 || echo "No fuzzy translations found."
Thom Wiggers's avatar
Thom Wiggers committed
34

Joren Vrancken's avatar
Joren Vrancken committed
35 36
django-tests:
  image: $PY38_IMAGE
Thom Wiggers's avatar
Thom Wiggers committed
37
  stage: test
Thom Wiggers's avatar
Thom Wiggers committed
38 39
  services:
    - postgres:latest
Thom Wiggers's avatar
Thom Wiggers committed
40 41
  before_script:
    - git log -1
Jelle Besseling's avatar
Jelle Besseling committed
42
    - poetry install --no-interaction
Thom Wiggers's avatar
Thom Wiggers committed
43
  script:
Thom Wiggers's avatar
Thom Wiggers committed
44
    - cd website
Jelle Besseling's avatar
Jelle Besseling committed
45 46 47
    - poetry run python manage.py check
    - poetry run python manage.py templatecheck --project-only
    - poetry run python manage.py makemigrations --no-input --check --dry-run
48
    - poetry run python -Wall -mcoverage run manage.py test
49
    - coverage report --fail-under=100 --omit registrations/urls.py registrations/**.py
50
    - coverage report --fail-under=100 --omit payments/urls.py payments/**.py
Thom Wiggers's avatar
Thom Wiggers committed
51
    - coverage report
52 53 54 55 56 57 58 59 60 61 62 63
  after_script:
    - cd website
    - coverage html --directory=covhtml --title="${CI_COMMIT_REF_SLUG} Coverage Report"
  artifacts:
    paths:
      - website/covhtml/

.sshsetup: &sshsetup
  before_script:
    - mkdir -p ~/.ssh
    - echo "$IVO_KNOWN_HOST" > ~/.ssh/known_hosts
    - echo "$COVERAGE_DEPLOY_SSH_KEY" > ~/.ssh/id_coverage
64 65
    - echo "$DOCS_DEPLOY_SSH_KEY" > ~/.ssh/id_docs
    - chmod 0600 ~/.ssh/id_*
66 67 68 69 70 71 72
    - apt-get update
    - apt-get install -y openssh-client

coverage deploy:
  stage: deploy
  image: debian:stretch
  dependencies:
Joren Vrancken's avatar
Joren Vrancken committed
73
    - django-tests
74
  environment:
75
    name: coverage/${CI_COMMIT_REF_NAME}
76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91
    url: https://coverage.technicie.nl/${CI_COMMIT_REF_SLUG}/
    on_stop: coverage remove
  <<: *sshsetup
  script:
    - |
      sftp -i ~/.ssh/id_coverage coveragewww@ivo.thalia.nu -b <<EOF
      -rm ${CI_COMMIT_REF_SLUG}/*
      -rmdir ${CI_COMMIT_REF_SLUG}
      put -r website/covhtml ${CI_COMMIT_REF_SLUG}
      EOF

coverage remove:
    stage: deploy
    when: manual
    image: debian:stretch
    environment:
92
        name: coverage/${CI_COMMIT_REF_NAME}
93 94 95 96 97 98 99 100 101 102
        action: stop
    variables:
        GIT_STRATEGY: none
    <<: *sshsetup
    script:
      - |
        sftp -i ~/.ssh/id_coverage coveragewww@ivo.thalia.nu -b <<EOF
        rm ${CI_COMMIT_REF_SLUG}/*
        rmdir ${CI_COMMIT_REF_SLUG}
        EOF
Thom Wiggers's avatar
Thom Wiggers committed
103

104
docs tests:
Thom Wiggers's avatar
Thom Wiggers committed
105
  stage: test
Joren Vrancken's avatar
Joren Vrancken committed
106
  image: $PY38_IMAGE
Thom Wiggers's avatar
Thom Wiggers committed
107 108
  before_script:
    # install django deps
Jelle Besseling's avatar
Jelle Besseling committed
109
    - poetry install --no-interaction --extras "docs"
Thom Wiggers's avatar
Thom Wiggers committed
110
  script:
Thom Wiggers's avatar
Thom Wiggers committed
111
    - echo "Building current docs"
Jelle Besseling's avatar
Jelle Besseling committed
112 113 114
    - cd docs
    - env -u GITLAB_CI poetry run make doctest
    - env -u GITLAB_CI poetry run sphinx-build -W . _build
Thom Wiggers's avatar
Thom Wiggers committed
115
    - echo "Checking if there are changes"
Jelle Besseling's avatar
Jelle Besseling committed
116
    - poetry run ./generate-apidocs.sh
Thom Wiggers's avatar
Thom Wiggers committed
117
    - git diff --exit-code
118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170
  artifacts:
    paths:
      - docs/_build

docs deploy:
  stage: deploy
  image: debian:stretch
  dependencies:
    - docs tests
  environment:
    name: docs/${CI_COMMIT_REF_NAME}
    url: https://docs.technicie.nl/${CI_COMMIT_REF_SLUG}/
    on_stop: docs remove
  <<: *sshsetup
  script:
    - |
      sftp -i ~/.ssh/id_docs docswww@ivo.thalia.nu -b <<EOF
      -rm ${CI_COMMIT_REF_SLUG}/*/*
      -rm ${CI_COMMIT_REF_SLUG}/*
      -rmdir ${CI_COMMIT_REF_SLUG}/_images
      -rmdir ${CI_COMMIT_REF_SLUG}/_modules
      -rmdir ${CI_COMMIT_REF_SLUG}/_sources
      -rmdir ${CI_COMMIT_REF_SLUG}/_static
      -rmdir ${CI_COMMIT_REF_SLUG}/doctest
      -rmdir ${CI_COMMIT_REF_SLUG}/doctrees
      -rmdir ${CI_COMMIT_REF_SLUG}
      -mkdir ${CI_COMMIT_REF_SLUG}
      put -r docs/_build/* ${CI_COMMIT_REF_SLUG}
      EOF

docs remove:
  stage: deploy
  when: manual
  image: debian:stretch
  environment:
    name: docs/${CI_COMMIT_REF_NAME}
    action: stop
  variables:
    GIT_STRATEGY: none
  <<: *sshsetup
  script:
    - |
      sftp -i ~/.ssh/id_docs docswww@ivo.thalia.nu -b <<EOF
      rm ${CI_COMMIT_REF_SLUG}/*/*
      rm ${CI_COMMIT_REF_SLUG}/*
      rmdir ${CI_COMMIT_REF_SLUG}/_images
      rmdir ${CI_COMMIT_REF_SLUG}/_modules
      rmdir ${CI_COMMIT_REF_SLUG}/_sources
      rmdir ${CI_COMMIT_REF_SLUG}/_static
      rmdir ${CI_COMMIT_REF_SLUG}/doctest
      rmdir ${CI_COMMIT_REF_SLUG}/doctrees
      rmdir ${CI_COMMIT_REF_SLUG}
      EOF
Thom Wiggers's avatar
Thom Wiggers committed
171

Thom Wiggers's avatar
Thom Wiggers committed
172 173 174 175 176 177 178
build docker image:
  stage: test
  services:
    - docker:dind
  image: thalia/docker-compose
  tags:
    - docker
179 180
  except:
    - tags
Thom Wiggers's avatar
Thom Wiggers committed
181
  before_script:
182
    - echo $DOCKER_REGISTRY_PASSWORD | docker login --username thaliawww --password-stdin registry.hub.docker.com
Thom Wiggers's avatar
Thom Wiggers committed
183
  script:
Joren Vrancken's avatar
Joren Vrancken committed
184
    - docker-compose config -q
185
    - docker-compose build --build-arg install_dev_requirements=$DEV_REQUIREMENTS --build-arg source_commit=$(git rev-parse HEAD) web
Thom Wiggers's avatar
Thom Wiggers committed
186 187 188
    - docker tag $DOCKER_LATEST $DOCKER_TAG
    - docker push $DOCKER_TAG
  variables:
189
    DEV_REQUIREMENTS: 1
190 191
    DOCKER_LATEST: registry.hub.docker.com/thalia/concrexit:latest
    DOCKER_TAG: registry.hub.docker.com/thalia/concrexit:$CI_COMMIT_SHA
Thom Wiggers's avatar
Thom Wiggers committed
192

193 194
build production docker image:
  extends: build docker image
Thom Wiggers's avatar
Thom Wiggers committed
195 196
  only:
    - tags
197 198
  except:
    - master
199
  after_script:
Thom Wiggers's avatar
Thom Wiggers committed
200
    - docker tag $DOCKER_TAG $DOCKER_TAG_PRODUCTION
201
    - docker tag $DOCKER_TAG $DOCKER_LATEST
Thom Wiggers's avatar
Thom Wiggers committed
202
    - docker push $DOCKER_TAG_PRODUCTION
203
    - docker push $DOCKER_LATEST    
Thom Wiggers's avatar
Thom Wiggers committed
204
  variables:
205
    DOCKER_TAG_PRODUCTION: registry.hub.docker.com/thalia/concrexit:$CI_COMMIT_TAG
206
    DEV_REQUIREMENTS: 0
Thom Wiggers's avatar
Thom Wiggers committed
207

208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246
.reviewsetup:
  when: "manual"
  image: "python:latest"
  variables:
      AWS_DEFAULT_REGION: "eu-west-1"
  before_script:
    - "DEBIAN_FRONTEND=noninteractive apt-get --yes --quiet update"
    - "DEBIAN_FRONTEND=noninteractive apt-get --yes --quiet install jq"
    - "pip install awscli"

review create:
  extends: ".reviewsetup"
  stage: "deploy"
  environment:
    name: "review/${CI_COMMIT_REF_NAME}"
    url: "https://${CI_COMMIT_REF_SLUG}.public.review.technicie.nl/"
    on_stop: "review remove"
  script:
    - "username=$(head /dev/urandom | tr -dc 'a-z' | head -c 10)"
    - "password=$(head /dev/urandom | tr -dc 'a-zA-Z' | head -c 32)"
    - >-
      sed --in-place
      --expression "s/@version@/$CI_COMMIT_SHA/g"
      --expression "s/@username@/$username/g"
      --expression "s/@password@/$password/g"
      ./resources/ec2-bootstrap.sh
    - "resources/review-host-create.sh"
    - 'echo -e "The deployment is done. Please wait for the website to come up. You can login on https://${CI_COMMIT_REF_SLUG}.public.review.technicie.nl/ with:\nUsername:$username\nPassword:$password"'


review remove:
  extends: ".reviewsetup"
  stage: "deploy"
  environment:
    name: "review/${CI_COMMIT_REF_NAME}"
    action: "stop"
  script:
    - "resources/review-host-remove.sh"

Thom Wiggers's avatar
Thom Wiggers committed
247
cache:
248
  key: "$CI_JOB_NAME"
Thom Wiggers's avatar
Thom Wiggers committed
249
  paths:
Thom Wiggers's avatar
Thom Wiggers committed
250
    - "${PIP_CACHE_DIR}"