.gitlab-ci.yml 10.1 KB
Newer Older
Wietse Kuipers's avatar
Wietse Kuipers committed
1 2 3 4
variables:
  POSTGRES_DB: thalia
  POSTGRES_USER: postgres
  POSTGRES_PASSWORD: ""
5 6
  # https://hub.docker.com/r/thalia/python-thalia/
  # https://github.com/thaliawww/python-thalia
7 8
  # Should get auto-updated with the official 'python' repository
  # Installs:
Jelle Besseling's avatar
Jelle Besseling committed
9
  #  - pip: coverage, poetry
10
  #  - apt: ghostscript
Thom Wiggers's avatar
Thom Wiggers committed
11
  PY37_IMAGE: thalia/python-thalia:3.7
Sébastiaan Versteeg's avatar
Sébastiaan Versteeg committed
12
  PY38_IMAGE: thalia/python-thalia:3.8
Thom Wiggers's avatar
Thom Wiggers committed
13
  PIP_CACHE_DIR: "${CI_PROJECT_DIR}/pip-cache"
Wietse Kuipers's avatar
Wietse Kuipers committed
14

Thom Wiggers's avatar
Thom Wiggers committed
15 16 17 18
stages:
  - test
  - deploy

19
codestyle:
Thom Wiggers's avatar
Thom Wiggers committed
20
  stage: test
Sébastiaan Versteeg's avatar
Sébastiaan Versteeg committed
21
  image: $PY37_IMAGE
Jelle Besseling's avatar
Jelle Besseling committed
22 23
  before_script:
    - poetry install --no-interaction
Thom Wiggers's avatar
Thom Wiggers committed
24
  script:
Luko van der Maas's avatar
Luko van der Maas committed
25
    - black --check .
26
    # Check for obsolete translations in .po files (starting with `#~`).
27
    - cd website
28
    - grep --include="*.po" --files-with-matches --recursive "^#~" && exit 1 || echo "No obsolete translations found."
29 30 31 32 33 34
    # Check for untranslated strings in .po files
    - empty_strings=$(sed '$a\\' **/locale/nl/LC_MESSAGES/django.po | tac | sed '/^$/N;/\nmsgstr ""$/,/^msgid/!d' | tac)
    - empty_strings+=$(sed '$a\\' locale/nl/LC_MESSAGES/django.po | tac | sed '/^$/N;/\nmsgstr ""$/,/^msgid/!d' | tac)
    - if [[ $empty_strings ]]; then echo $empty_strings && exit 1; else echo "No untranslated strings found."; fi
    # Check for fuzzy translations in .po files
    - grep --include="*.po" --files-with-matches --recursive "#, fuzzy" && exit 1 || echo "No fuzzy translations found."
Thom Wiggers's avatar
Thom Wiggers committed
35 36

.djangotest: &djangotest
Thom Wiggers's avatar
Thom Wiggers committed
37
  stage: test
Thom Wiggers's avatar
Thom Wiggers committed
38 39
  services:
    - postgres:latest
Thom Wiggers's avatar
Thom Wiggers committed
40 41
  before_script:
    - git log -1
Jelle Besseling's avatar
Jelle Besseling committed
42
    - poetry install --no-interaction
Thom Wiggers's avatar
Thom Wiggers committed
43
  script:
Thom Wiggers's avatar
Thom Wiggers committed
44
    - cd website
Jelle Besseling's avatar
Jelle Besseling committed
45 46 47
    - poetry run python manage.py check
    - poetry run python manage.py templatecheck --project-only
    - poetry run python manage.py makemigrations --no-input --check --dry-run
48
    - poetry run python -Wall -mcoverage run manage.py test
49 50
    - coverage report --fail-under=100 --omit registrations/urls.py registrations/**.py
    - coverage report --fail-under=97.32 --omit payments/urls.py payments/**.py
Thom Wiggers's avatar
Thom Wiggers committed
51
    - coverage report
Thom Wiggers's avatar
Thom Wiggers committed
52

Sébastiaan Versteeg's avatar
Sébastiaan Versteeg committed
53
python37-django22:
Thom Wiggers's avatar
Thom Wiggers committed
54
  <<: *djangotest
Thom Wiggers's avatar
Thom Wiggers committed
55
  image: $PY37_IMAGE
56 57 58 59 60 61 62
  after_script:
    - cd website
    - coverage html --directory=covhtml --title="${CI_COMMIT_REF_SLUG} Coverage Report"
  artifacts:
    paths:
      - website/covhtml/

Sébastiaan Versteeg's avatar
Sébastiaan Versteeg committed
63 64 65 66 67
python38-django22:
  <<: *djangotest
  image: $PY38_IMAGE
  allow_failure: true

68 69 70 71 72
.sshsetup: &sshsetup
  before_script:
    - mkdir -p ~/.ssh
    - echo "$IVO_KNOWN_HOST" > ~/.ssh/known_hosts
    - echo "$COVERAGE_DEPLOY_SSH_KEY" > ~/.ssh/id_coverage
73 74
    - echo "$DOCS_DEPLOY_SSH_KEY" > ~/.ssh/id_docs
    - chmod 0600 ~/.ssh/id_*
75 76 77 78 79 80 81
    - apt-get update
    - apt-get install -y openssh-client

coverage deploy:
  stage: deploy
  image: debian:stretch
  dependencies:
Sébastiaan Versteeg's avatar
Sébastiaan Versteeg committed
82
    - python37-django22
83
  environment:
84
    name: coverage/${CI_COMMIT_REF_NAME}
85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100
    url: https://coverage.technicie.nl/${CI_COMMIT_REF_SLUG}/
    on_stop: coverage remove
  <<: *sshsetup
  script:
    - |
      sftp -i ~/.ssh/id_coverage coveragewww@ivo.thalia.nu -b <<EOF
      -rm ${CI_COMMIT_REF_SLUG}/*
      -rmdir ${CI_COMMIT_REF_SLUG}
      put -r website/covhtml ${CI_COMMIT_REF_SLUG}
      EOF

coverage remove:
    stage: deploy
    when: manual
    image: debian:stretch
    environment:
101
        name: coverage/${CI_COMMIT_REF_NAME}
102 103 104 105 106 107 108 109 110 111
        action: stop
    variables:
        GIT_STRATEGY: none
    <<: *sshsetup
    script:
      - |
        sftp -i ~/.ssh/id_coverage coveragewww@ivo.thalia.nu -b <<EOF
        rm ${CI_COMMIT_REF_SLUG}/*
        rmdir ${CI_COMMIT_REF_SLUG}
        EOF
Thom Wiggers's avatar
Thom Wiggers committed
112

113
docs tests:
Thom Wiggers's avatar
Thom Wiggers committed
114
  stage: test
Sébastiaan Versteeg's avatar
Sébastiaan Versteeg committed
115
  image: $PY37_IMAGE
Thom Wiggers's avatar
Thom Wiggers committed
116 117
  before_script:
    # install django deps
Jelle Besseling's avatar
Jelle Besseling committed
118
    - poetry install --no-interaction --extras "docs"
Thom Wiggers's avatar
Thom Wiggers committed
119
  script:
Thom Wiggers's avatar
Thom Wiggers committed
120
    - echo "Building current docs"
Jelle Besseling's avatar
Jelle Besseling committed
121 122 123
    - cd docs
    - env -u GITLAB_CI poetry run make doctest
    - env -u GITLAB_CI poetry run sphinx-build -W . _build
Thom Wiggers's avatar
Thom Wiggers committed
124
    - echo "Checking if there are changes"
Jelle Besseling's avatar
Jelle Besseling committed
125
    - poetry run ./generate-apidocs.sh
Thom Wiggers's avatar
Thom Wiggers committed
126
    - git diff --exit-code
127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179
  artifacts:
    paths:
      - docs/_build

docs deploy:
  stage: deploy
  image: debian:stretch
  dependencies:
    - docs tests
  environment:
    name: docs/${CI_COMMIT_REF_NAME}
    url: https://docs.technicie.nl/${CI_COMMIT_REF_SLUG}/
    on_stop: docs remove
  <<: *sshsetup
  script:
    - |
      sftp -i ~/.ssh/id_docs docswww@ivo.thalia.nu -b <<EOF
      -rm ${CI_COMMIT_REF_SLUG}/*/*
      -rm ${CI_COMMIT_REF_SLUG}/*
      -rmdir ${CI_COMMIT_REF_SLUG}/_images
      -rmdir ${CI_COMMIT_REF_SLUG}/_modules
      -rmdir ${CI_COMMIT_REF_SLUG}/_sources
      -rmdir ${CI_COMMIT_REF_SLUG}/_static
      -rmdir ${CI_COMMIT_REF_SLUG}/doctest
      -rmdir ${CI_COMMIT_REF_SLUG}/doctrees
      -rmdir ${CI_COMMIT_REF_SLUG}
      -mkdir ${CI_COMMIT_REF_SLUG}
      put -r docs/_build/* ${CI_COMMIT_REF_SLUG}
      EOF

docs remove:
  stage: deploy
  when: manual
  image: debian:stretch
  environment:
    name: docs/${CI_COMMIT_REF_NAME}
    action: stop
  variables:
    GIT_STRATEGY: none
  <<: *sshsetup
  script:
    - |
      sftp -i ~/.ssh/id_docs docswww@ivo.thalia.nu -b <<EOF
      rm ${CI_COMMIT_REF_SLUG}/*/*
      rm ${CI_COMMIT_REF_SLUG}/*
      rmdir ${CI_COMMIT_REF_SLUG}/_images
      rmdir ${CI_COMMIT_REF_SLUG}/_modules
      rmdir ${CI_COMMIT_REF_SLUG}/_sources
      rmdir ${CI_COMMIT_REF_SLUG}/_static
      rmdir ${CI_COMMIT_REF_SLUG}/doctest
      rmdir ${CI_COMMIT_REF_SLUG}/doctrees
      rmdir ${CI_COMMIT_REF_SLUG}
      EOF
Thom Wiggers's avatar
Thom Wiggers committed
180

Thom Wiggers's avatar
Thom Wiggers committed
181 182 183 184 185 186 187
build docker image:
  stage: test
  services:
    - docker:dind
  image: thalia/docker-compose
  tags:
    - docker
188 189
  except:
    - tags
Thom Wiggers's avatar
Thom Wiggers committed
190
  before_script:
191
    - echo $DOCKER_REGISTRY_PASSWORD | docker login --username thaliawww --password-stdin registry.hub.docker.com
Thom Wiggers's avatar
Thom Wiggers committed
192
  script:
Joren Vrancken's avatar
Joren Vrancken committed
193
    - docker-compose config -q
194
    - docker-compose build --build-arg install_dev_requirements=$DEV_REQUIREMENTS --build-arg source_commit=$(git rev-parse HEAD) web
Thom Wiggers's avatar
Thom Wiggers committed
195 196 197
    - docker tag $DOCKER_LATEST $DOCKER_TAG
    - docker push $DOCKER_TAG
  variables:
198
    DEV_REQUIREMENTS: 1
199 200
    DOCKER_LATEST: registry.hub.docker.com/thalia/concrexit:latest
    DOCKER_TAG: registry.hub.docker.com/thalia/concrexit:$CI_COMMIT_SHA
Thom Wiggers's avatar
Thom Wiggers committed
201

Jelle Besseling's avatar
Jelle Besseling committed
202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299
.reviewsetup:
  when: manual
  image: python:latest
  before_script:
    - apt-get update
    - apt-get install -y jq
    - pip install awscli
    - >-
      instanceids=$(
      aws --region eu-west-1 ec2 describe-instances
      --filters "Name=tag:Name,Values=concrexit-review-${CI_COMMIT_REF_SLUG}"
      | jq --raw-output '.Reservations|map(.Instances[0].InstanceId)|join(" ")'
      )
    - aws --region eu-west-1 ec2 terminate-instances --instance-ids ${instanceids} || true

review:
  stage: deploy
  environment:
    name: review/${CI_COMMIT_REF_NAME}
    url: https://${CI_COMMIT_REF_SLUG}.review.technicie.nl/
    on_stop: review remove
  extends: .reviewsetup
  script:
    - username=$(head /dev/urandom | tr -dc 'a-z' | head -c 10)
    - password=$(head /dev/urandom | tr -dc 'a-zA-Z' | head -c 32)
    - echo -e "When the deployment is done, you can login with:\n$username\n$password"
    - >-
      sed -i -e "s/@version@/$CI_COMMIT_SHA/g"
      -e "s/@username@/$username/g"
      -e "s/@password@/$password/g"
      ./resources/ec2-bootstrap.sh
    - >-
      instanceid=$(
      aws --region eu-west-1 ec2 run-instances
      --count 1
      --instance-type t2.micro
      --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=concrexit-review-${CI_COMMIT_REF_SLUG}}]"
      --launch-template LaunchTemplateId=lt-03762fc23450c2471,Version=1
      --user-data file://resources/ec2-bootstrap.sh
      | jq --raw-output '.Instances[0].InstanceId'
      )
    - aws --region eu-west-1 ec2 wait instance-running --instance-ids ${instanceid}
    - ipaddress=$(aws --region eu-west-1 ec2 describe-instances --instance-ids ${instanceid} | jq --raw-output '.Reservations[0].Instances[0].PublicIpAddress')
    - |
      cat > add-record.json <<EOF
      {
        "Comment": "CREATE review deployment record",
        "Changes": [
          {
            "Action": "CREATE",
            "ResourceRecordSet": {
              "Name": "${CI_COMMIT_REF_SLUG}.review.technicie.nl.reviewhost",
              "Type": "A",
              "TTL": 10,
              "ResourceRecords": [{"Value": "${ipaddress}"}]
            }
          }
        ]

      }
      EOF
    - |
      cat > change-record.json <<EOF
      {
        "Comment": "CHANGE review deployment record",
        "Changes": [
          {
            "Action": "UPSERT",
            "ResourceRecordSet": {
              "Name": "${CI_COMMIT_REF_SLUG}.review.technicie.nl.reviewhost",
              "Type": "A",
              "TTL": 10,
              "ResourceRecords": [{"Value": "${ipaddress}"}]
            }
          }
        ]
      }
      EOF
    - >-
      changeinfoid=$(
      (
      aws --region eu-west-1 route53 change-resource-record-sets
      --hosted-zone-id Z072013523EW763CDQ8K4
      --change-batch file://add-record.json
      ||
      aws --region eu-west-1 route53 change-resource-record-sets
      --hosted-zone-id Z072013523EW763CDQ8K4
      --change-batch file://change-record.json
      )
      | jq --raw-output '.ChangeInfo.Id'
      )
    - aws --region eu-west-1 route53 wait resource-record-sets-changed --id ${changeinfoid}

review remove:
  stage: deploy
  environment:
    name: review/${CI_COMMIT_REF_NAME}
    action: stop
300 301
  variables:
    GIT_STRATEGY: none
Jelle Besseling's avatar
Jelle Besseling committed
302 303 304 305 306 307 308 309 310 311 312
  extends: .reviewsetup
  script:
    - >-
      aws --region eu-west-1 route53 list-resource-record-sets
      --hosted-zone-id Z072013523EW763CDQ8K4
      --query "ResourceRecordSets[?Name == '${CI_COMMIT_REF_SLUG}.review.technicie.nl.']"
      |
      jq '{"Comment": "DELETE review deployment record", "Changes": map({"Action": "DELETE", "ResourceRecordSet": .})}'
      > remove-record.json
    - aws --region eu-west-1 route53 change-resource-record-sets --hosted-zone-id Z072013523EW763CDQ8K4 --change-batch file://remove-record.json || true

313 314
build production docker image:
  extends: build docker image
Thom Wiggers's avatar
Thom Wiggers committed
315 316
  only:
    - tags
317 318
  except:
    - master
319
  after_script:
Thom Wiggers's avatar
Thom Wiggers committed
320
    - docker tag $DOCKER_TAG $DOCKER_TAG_PRODUCTION
321
    - docker tag $DOCKER_TAG $DOCKER_LATEST
Thom Wiggers's avatar
Thom Wiggers committed
322
    - docker push $DOCKER_TAG_PRODUCTION
323
    - docker push $DOCKER_LATEST    
Thom Wiggers's avatar
Thom Wiggers committed
324
  variables:
325
    DOCKER_TAG_PRODUCTION: registry.hub.docker.com/thalia/concrexit:$CI_COMMIT_TAG
326
    DEV_REQUIREMENTS: 0
Thom Wiggers's avatar
Thom Wiggers committed
327

Thom Wiggers's avatar
Thom Wiggers committed
328
cache:
329
  key: "$CI_JOB_NAME"
Thom Wiggers's avatar
Thom Wiggers committed
330
  paths:
Thom Wiggers's avatar
Thom Wiggers committed
331
    - "${PIP_CACHE_DIR}"
Thom Wiggers's avatar
Thom Wiggers committed
332 333

# vim: set sw=2 ts=2 et :