.gitlab-ci.yml 7.58 KB
Newer Older
Markus Scheidgen's avatar
Markus Scheidgen committed
1
# default installed image for docker executor is: python:3.6
2
3
# using an image that can do git, docker, docker-compose
image: youpy/docker-compose-git
4
5
6
7
8

# Uncomment the next lines, to run each pipline/job in its own docker environment.
# Otherwise, it will use the docker of the gitlab runner host (e.g. enc-preprocessing...).
# This will give it access to a persitent layer cache, which will not be available
# with the docker service.
9
10
# services:
#   - docker:dind
Markus Scheidgen's avatar
Markus Scheidgen committed
11
12

stages:
Markus Scheidgen's avatar
Markus Scheidgen committed
13
  - build
14
  - test
15
  - integration
16
  - release
17
  - deploy
18

19
variables:
Markus Scheidgen's avatar
Markus Scheidgen committed
20
  TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:${CI_COMMIT_REF_NAME}
Markus Scheidgen's avatar
Markus Scheidgen committed
21
  RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest
Markus Scheidgen's avatar
Markus Scheidgen committed
22
  FRONTEND_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:${CI_COMMIT_REF_NAME}
Markus Scheidgen's avatar
Markus Scheidgen committed
23
  FRONTEND_RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:latest
Markus Scheidgen's avatar
Markus Scheidgen committed
24
  RAWAPI_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/rawapi:${CI_COMMIT_REF_NAME}
Markus Scheidgen's avatar
Markus Scheidgen committed
25
  RAWAPI_RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/rawapi:latest
Markus Scheidgen's avatar
Markus Scheidgen committed
26

27
28
29
  KUBECONFIG: /etc/deploy/config
  STAGING_NAMESPACE: nomad-fairdi

Markus Scheidgen's avatar
Markus Scheidgen committed
30
31
build:
  stage: build
32
33
  before_script:
    - git submodule sync
34
    - git submodule update --init
Markus Scheidgen's avatar
Markus Scheidgen committed
35
  script:
36
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
Markus Scheidgen's avatar
Markus Scheidgen committed
37
    - docker build --no-cache -t $TEST_IMAGE .
Markus Scheidgen's avatar
Markus Scheidgen committed
38
    - docker push $TEST_IMAGE
39
40
41
  except:
    - /^dev-.*$/

Markus Scheidgen's avatar
Markus Scheidgen committed
42
43
44
45
46
47

buildgui:
  stage: build
  script:
    - cd gui
    - ./gitinfo.sh
Markus Scheidgen's avatar
Markus Scheidgen committed
48
    - ls -la src/gitinfo.json
Markus Scheidgen's avatar
Markus Scheidgen committed
49
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
Markus Scheidgen's avatar
Markus Scheidgen committed
50
    - docker build -t $FRONTEND_TEST_IMAGE .
Markus Scheidgen's avatar
Markus Scheidgen committed
51
    - docker push $FRONTEND_TEST_IMAGE
52
53
54
  except:
    - /^dev-.*$/

55

Markus Scheidgen's avatar
Markus Scheidgen committed
56
57
58
59
buildrawapi:
  stage: build
  script:
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
60
    - docker build -t $RAWAPI_TEST_IMAGE -f rawapi.Dockerfile .
Markus Scheidgen's avatar
Markus Scheidgen committed
61
    - docker push $RAWAPI_TEST_IMAGE
62
63
  only:
    - rawapi
Markus Scheidgen's avatar
Markus Scheidgen committed
64
65


66
linting:
67
  stage: test
Markus Scheidgen's avatar
Markus Scheidgen committed
68
  image: $TEST_IMAGE
69
  script:
70
    - cd /app
Markus Scheidgen's avatar
Markus Scheidgen committed
71
72
73
    - python -m pycodestyle --ignore=E501,E701 nomad tests
    - python -m pylint --load-plugins=pylint_mongoengine nomad tests
    - python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests
74
  except:
75
76
    variables:
      - $CI_COMMIT_REF_NAME =~ /^dev-.*$/
77
      - $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
78
79

tests:
Markus Scheidgen's avatar
Markus Scheidgen committed
80
  stage: test
Markus Scheidgen's avatar
Markus Scheidgen committed
81
  image: $TEST_IMAGE
82
  services:
83
    - postgres:latest
84
85
86
87
88
89
90
91
92
93
      # this will cause a warning, as the gitlab ci runner health check will test the wrong
      # port on rabbitmq container:
      # https://gitlab.com/gitlab-org/gitlab-runner/issues/3163
    - rabbitmq
    - name: docker.elastic.co/elasticsearch/elasticsearch:6.3.2
      alias: elastic
      # fix issue with running elastic in gitlab ci runner:
      # https://gitlab.com/gitlab-org/gitlab-ce/issues/42214
      command: [ "bin/elasticsearch", "-Ediscovery.type=single-node" ]
  variables:
94
95
96
    POSTGRES_DB: nomad
    POSTGRES_USER: postgres
    POSTGRES_PASSWORD: nomad
97
98
99
100
    RABBITMQ_ERLANG_COOKIE: SWQOKODSQALRPCLNMEQG
    RABBITMQ_DEFAULT_USER: rabbitmq
    RABBITMQ_DEFAULT_PASS: rabbitmq
    RABBITMQ_DEFAULT_VHOST: /
101
    NOMAD_COE_REPO_DB_HOST: postgres
102
103
104
    NOMAD_RABBITMQ_HOST: rabbitmq
    NOMAD_ELASTIC_HOST: elastic
    NOMAD_MONGO_HOST: mongo
Markus Scheidgen's avatar
Markus Scheidgen committed
105
  script:
106
    - cd /app
Markus Scheidgen's avatar
Markus Scheidgen committed
107
    - python -m pytest --cov=nomad -sv tests
108
  except:
109
110
    variables:
      - $CI_COMMIT_REF_NAME =~ /^dev-.*$/
111
      - $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
112

113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
# does currently not work, current GitLab CI runner does not network services with each other
# integration-tests:
#   stage: integration
#   services:
#     - postgres:latest
#     - mongo:latest
#       # this will cause a warning, as the gitlab ci runner health check will test the wrong
#       # port on rabbitmq container:
#       # https://gitlab.com/gitlab-org/gitlab-runner/issues/3163
#     - rabbitmq
#     - name: docker.elastic.co/elasticsearch/elasticsearch:6.3.2
#       alias: elastic
#       # fix issue with running elastic in gitlab ci runner:
#       # https://gitlab.com/gitlab-org/gitlab-ce/issues/42214
#       command: [ "bin/elasticsearch", "-Ediscovery.type=single-node" ]
#     - name: $TEST_IMAGE
#       alias: api
#       command: ["python", "-m", "nomad.client", "run", "api"]
#     - name: $TEST_IMAGE
#       alias: worker
#       command: ["python", "-m", "nomad.client", "run", "worker"]
#   variables:
#     POSTGRES_DB: nomad
#     POSTGRES_USER: postgres
#     POSTGRES_PASSWORD: nomad
#     RABBITMQ_ERLANG_COOKIE: SWQOKODSQALRPCLNMEQG
#     RABBITMQ_DEFAULT_USER: rabbitmq
#     RABBITMQ_DEFAULT_PASS: rabbitmq
#     RABBITMQ_DEFAULT_VHOST: /
#     NOMAD_COE_REPO_DB_HOST: postgres
#     NOMAD_RABBITMQ_HOST: rabbitmq
#     NOMAD_ELASTIC_HOST: elastic
#     NOMAD_MONGO_HOST: mongo
#   script:
#     - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
#     - integration/test_integration.sh
#   except:
#     - /^dev-.*$/
151
152
153
154
155

release:
  stage: release
  script:
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
Markus Scheidgen's avatar
Markus Scheidgen committed
156
157
158
159
160
161
    - docker pull $TEST_IMAGE
    - docker tag $TEST_IMAGE $RELEASE_IMAGE
    - docker push $RELEASE_IMAGE
    - docker pull $FRONTEND_TEST_IMAGE
    - docker tag $FRONTEND_TEST_IMAGE $FRONTEND_RELEASE_IMAGE
    - docker push $FRONTEND_RELEASE_IMAGE
Markus Scheidgen's avatar
Markus Scheidgen committed
162
163
164
  except:
    - /^dev-.*$/
  when: manual
165

Markus Scheidgen's avatar
Markus Scheidgen committed
166
release_rawapi:
167
168
169
170
171
172
  stage: release
  script:
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
    - docker pull $RAWAPI_TEST_IMAGE
    - docker tag $RAWAPI_TEST_IMAGE $RAWAPI_RELEASE_IMAGE
    - docker push $RAWAPI_RELEASE_IMAGE
173
174
  only:
    - rawapi
175
176
  when: manual

Markus Scheidgen's avatar
Markus Scheidgen committed
177
deploy_rawapi:
Markus Scheidgen's avatar
Markus Scheidgen committed
178
  stage: deploy
179
180
181
182
183
184
185
  script:
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
    - cd ops/docker-compose/rawapi
    # secret .env file is stored on the "production" machine and is bind mounted into the
    # runner container at /nomad/config/.env
    - cp /nomad/config/.rawapi_env .env
    - docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d
186
187
  only:
    - rawapi
188
  when: manual
189

Markus Scheidgen's avatar
Markus Scheidgen committed
190
deploy:
191
192
193
194
195
  stage: deploy
  image: dtzar/helm-kubectl
  before_script:
    - mkdir -p /etc/deploy
    # kube_config is a CI/CD variable set in GitLab GUI
196
    - echo $CI_KUBE_CONFIG | base64 -d > /etc/deploy/config
197
198
199
200
201
202
    - helm init --client-only
    - helm repo add stable https://kubernetes-charts.storage.googleapis.com/
    - helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com/
    - helm repo update
  script:
    - cd ops/helm/nomad
203
    - export KUBECONFIG=/etc/deploy/config
204
    - helm dep build
Markus Scheidgen's avatar
Markus Scheidgen committed
205
206
207
    - export NOMAD_VERSION="$(grep "version" Chart.yaml | cut -d" " -f2)"
    - export NUMERIC_VERSION="$(echo ${NOMAD_VERSION} | cut -d"." -f1-2)"
    - export RELEASE_NAME="nomad-v${NOMAD_VERSION//./-}"
208
    - export DEPLOYS=$(helm ls | grep $RELEASE_NAME | wc -l)
Markus Scheidgen's avatar
Markus Scheidgen committed
209
210
211
    - export EXTERNAL_PATH="/fairdi/nomad/v${NOMAD_VERSION}"
    - export DBNAME="fairdi_nomad_v${NOMAD_VERSION//./_}"
    - export FILES_PATH="/scratch/nomad-fair/fs/nomad_v${NOMAD_VERSION}"
Markus Scheidgen's avatar
Markus Scheidgen committed
212
213
    - if [ ${DEPLOYS}  -eq 0 ]; then
        helm install --name=${RELEASE_NAME} . --namespace=${STAGING_NAMESPACE}
214
          --set api.disableReset="false"
Markus Scheidgen's avatar
Markus Scheidgen committed
215
          --set proxy.nodePort="300${NUMERIC_VERSION//./}"
Markus Scheidgen's avatar
Markus Scheidgen committed
216
217
          --set proxy.external.path=${EXTERNAL_PATH}
          --set dbname=${DBNAME}
218
219
          --set worker.replicas=1
          --set worker.memrequest=32
Markus Scheidgen's avatar
Markus Scheidgen committed
220
221
          --set volumes.files=${FILES_PATH};
      else
222
        helm upgrade ${RELEASE_NAME} . --namespace=${STAGING_NAMESPACE} --recreate-pods;
Markus Scheidgen's avatar
Markus Scheidgen committed
223
      fi
Markus Scheidgen's avatar
Markus Scheidgen committed
224
225
  except:
    - /^dev-.*$/
226
  when: manual