.gitlab-ci.yml 7.4 KB
Newer Older
Markus Scheidgen's avatar
Markus Scheidgen committed
1
# default installed image for docker executor is: python:3.6
2
3
# using an image that can do git, docker, docker-compose
image: youpy/docker-compose-git
4
5
6
7
8

# Uncomment the next lines, to run each pipline/job in its own docker environment.
# Otherwise, it will use the docker of the gitlab runner host (e.g. enc-preprocessing...).
# This will give it access to a persitent layer cache, which will not be available
# with the docker service.
9
10
# services:
#   - docker:dind
Markus Scheidgen's avatar
Markus Scheidgen committed
11
12

stages:
Markus Scheidgen's avatar
Markus Scheidgen committed
13
  - build
14
  - test
15
  - integration
16
  - release
17
  - deploy
18

19
variables:
Markus Scheidgen's avatar
Markus Scheidgen committed
20
  TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:${CI_COMMIT_REF_NAME}
Markus Scheidgen's avatar
Markus Scheidgen committed
21
  RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest
Markus Scheidgen's avatar
Markus Scheidgen committed
22
  FRONTEND_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:${CI_COMMIT_REF_NAME}
Markus Scheidgen's avatar
Markus Scheidgen committed
23
  FRONTEND_RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:latest
Markus Scheidgen's avatar
Markus Scheidgen committed
24
  RAWAPI_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/rawapi:${CI_COMMIT_REF_NAME}
Markus Scheidgen's avatar
Markus Scheidgen committed
25
  RAWAPI_RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/rawapi:latest
Markus Scheidgen's avatar
Markus Scheidgen committed
26

27
28
29
  KUBECONFIG: /etc/deploy/config
  STAGING_NAMESPACE: nomad-fairdi

Markus Scheidgen's avatar
Markus Scheidgen committed
30
31
build:
  stage: build
32
33
  before_script:
    - git submodule sync
34
    - git submodule update --init
Markus Scheidgen's avatar
Markus Scheidgen committed
35
  script:
36
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
Markus Scheidgen's avatar
Markus Scheidgen committed
37
    - docker build --no-cache -t $TEST_IMAGE .
Markus Scheidgen's avatar
Markus Scheidgen committed
38
    - docker push $TEST_IMAGE
39
40
41
  except:
    - /^dev-.*$/

Markus Scheidgen's avatar
Markus Scheidgen committed
42
43
44
45
46
47

buildgui:
  stage: build
  script:
    - cd gui
    - ./gitinfo.sh
Markus Scheidgen's avatar
Markus Scheidgen committed
48
    - ls -la src/gitinfo.json
Markus Scheidgen's avatar
Markus Scheidgen committed
49
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
Markus Scheidgen's avatar
Markus Scheidgen committed
50
    - docker build -t $FRONTEND_TEST_IMAGE .
Markus Scheidgen's avatar
Markus Scheidgen committed
51
    - docker push $FRONTEND_TEST_IMAGE
52
53
54
  except:
    - /^dev-.*$/

55

Markus Scheidgen's avatar
Markus Scheidgen committed
56
57
58
59
buildrawapi:
  stage: build
  script:
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
60
    - docker build -t $RAWAPI_TEST_IMAGE -f rawapi.Dockerfile .
Markus Scheidgen's avatar
Markus Scheidgen committed
61
    - docker push $RAWAPI_TEST_IMAGE
62
63
  only:
    - rawapi
Markus Scheidgen's avatar
Markus Scheidgen committed
64
65


66
linting:
67
  stage: test
Markus Scheidgen's avatar
Markus Scheidgen committed
68
  image: $TEST_IMAGE
69
  script:
70
    - cd /app
Markus Scheidgen's avatar
Markus Scheidgen committed
71
72
73
    - python -m pycodestyle --ignore=E501,E701 nomad tests
    - python -m pylint --load-plugins=pylint_mongoengine nomad tests
    - python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests
74
75
  except:
    - /^dev-.*$/
76
77

tests:
Markus Scheidgen's avatar
Markus Scheidgen committed
78
  stage: test
Markus Scheidgen's avatar
Markus Scheidgen committed
79
  image: $TEST_IMAGE
80
  services:
81
    - postgres:latest
82
83
84
85
86
87
88
89
90
91
      # this will cause a warning, as the gitlab ci runner health check will test the wrong
      # port on rabbitmq container:
      # https://gitlab.com/gitlab-org/gitlab-runner/issues/3163
    - rabbitmq
    - name: docker.elastic.co/elasticsearch/elasticsearch:6.3.2
      alias: elastic
      # fix issue with running elastic in gitlab ci runner:
      # https://gitlab.com/gitlab-org/gitlab-ce/issues/42214
      command: [ "bin/elasticsearch", "-Ediscovery.type=single-node" ]
  variables:
92
93
94
    POSTGRES_DB: nomad
    POSTGRES_USER: postgres
    POSTGRES_PASSWORD: nomad
95
96
97
98
    RABBITMQ_ERLANG_COOKIE: SWQOKODSQALRPCLNMEQG
    RABBITMQ_DEFAULT_USER: rabbitmq
    RABBITMQ_DEFAULT_PASS: rabbitmq
    RABBITMQ_DEFAULT_VHOST: /
99
    NOMAD_COE_REPO_DB_HOST: postgres
100
101
102
    NOMAD_RABBITMQ_HOST: rabbitmq
    NOMAD_ELASTIC_HOST: elastic
    NOMAD_MONGO_HOST: mongo
Markus Scheidgen's avatar
Markus Scheidgen committed
103
  script:
104
    - cd /app
Markus Scheidgen's avatar
Markus Scheidgen committed
105
    - python -m pytest --cov=nomad -sv tests
106
107
108
  except:
    - /^dev-.*$/

109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
# does currently not work, current GitLab CI runner does not network services with each other
# integration-tests:
#   stage: integration
#   services:
#     - postgres:latest
#     - mongo:latest
#       # this will cause a warning, as the gitlab ci runner health check will test the wrong
#       # port on rabbitmq container:
#       # https://gitlab.com/gitlab-org/gitlab-runner/issues/3163
#     - rabbitmq
#     - name: docker.elastic.co/elasticsearch/elasticsearch:6.3.2
#       alias: elastic
#       # fix issue with running elastic in gitlab ci runner:
#       # https://gitlab.com/gitlab-org/gitlab-ce/issues/42214
#       command: [ "bin/elasticsearch", "-Ediscovery.type=single-node" ]
#     - name: $TEST_IMAGE
#       alias: api
#       command: ["python", "-m", "nomad.client", "run", "api"]
#     - name: $TEST_IMAGE
#       alias: worker
#       command: ["python", "-m", "nomad.client", "run", "worker"]
#   variables:
#     POSTGRES_DB: nomad
#     POSTGRES_USER: postgres
#     POSTGRES_PASSWORD: nomad
#     RABBITMQ_ERLANG_COOKIE: SWQOKODSQALRPCLNMEQG
#     RABBITMQ_DEFAULT_USER: rabbitmq
#     RABBITMQ_DEFAULT_PASS: rabbitmq
#     RABBITMQ_DEFAULT_VHOST: /
#     NOMAD_COE_REPO_DB_HOST: postgres
#     NOMAD_RABBITMQ_HOST: rabbitmq
#     NOMAD_ELASTIC_HOST: elastic
#     NOMAD_MONGO_HOST: mongo
#   script:
#     - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
#     - integration/test_integration.sh
#   except:
#     - /^dev-.*$/
147
148
149
150
151

release:
  stage: release
  script:
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
Markus Scheidgen's avatar
Markus Scheidgen committed
152
153
154
155
156
157
    - docker pull $TEST_IMAGE
    - docker tag $TEST_IMAGE $RELEASE_IMAGE
    - docker push $RELEASE_IMAGE
    - docker pull $FRONTEND_TEST_IMAGE
    - docker tag $FRONTEND_TEST_IMAGE $FRONTEND_RELEASE_IMAGE
    - docker push $FRONTEND_RELEASE_IMAGE
Markus Scheidgen's avatar
Markus Scheidgen committed
158
159
160
  except:
    - /^dev-.*$/
  when: manual
161

Markus Scheidgen's avatar
Markus Scheidgen committed
162
release_rawapi:
163
164
165
166
167
168
  stage: release
  script:
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
    - docker pull $RAWAPI_TEST_IMAGE
    - docker tag $RAWAPI_TEST_IMAGE $RAWAPI_RELEASE_IMAGE
    - docker push $RAWAPI_RELEASE_IMAGE
169
170
  only:
    - rawapi
171
172
  when: manual

Markus Scheidgen's avatar
Markus Scheidgen committed
173
deploy_rawapi:
Markus Scheidgen's avatar
Markus Scheidgen committed
174
  stage: deploy
175
176
177
178
179
180
181
  script:
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
    - cd ops/docker-compose/rawapi
    # secret .env file is stored on the "production" machine and is bind mounted into the
    # runner container at /nomad/config/.env
    - cp /nomad/config/.rawapi_env .env
    - docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d
182
183
  only:
    - rawapi
184
  when: manual
185

Markus Scheidgen's avatar
Markus Scheidgen committed
186
deploy:
187
188
189
190
191
  stage: deploy
  image: dtzar/helm-kubectl
  before_script:
    - mkdir -p /etc/deploy
    # kube_config is a CI/CD variable set in GitLab GUI
192
    - echo $CI_KUBE_CONFIG | base64 -d > /etc/deploy/config
193
194
195
196
197
198
    - helm init --client-only
    - helm repo add stable https://kubernetes-charts.storage.googleapis.com/
    - helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com/
    - helm repo update
  script:
    - cd ops/helm/nomad
199
    - export KUBECONFIG=/etc/deploy/config
200
    - helm dep build
Markus Scheidgen's avatar
Markus Scheidgen committed
201
202
203
    - export NOMAD_VERSION="$(grep "version" Chart.yaml | cut -d" " -f2)"
    - export NUMERIC_VERSION="$(echo ${NOMAD_VERSION} | cut -d"." -f1-2)"
    - export RELEASE_NAME="nomad-v${NOMAD_VERSION//./-}"
204
    - export DEPLOYS=$(helm ls | grep $RELEASE_NAME | wc -l)
Markus Scheidgen's avatar
Markus Scheidgen committed
205
206
207
    - export EXTERNAL_PATH="/fairdi/nomad/v${NOMAD_VERSION}"
    - export DBNAME="fairdi_nomad_v${NOMAD_VERSION//./_}"
    - export FILES_PATH="/scratch/nomad-fair/fs/nomad_v${NOMAD_VERSION}"
Markus Scheidgen's avatar
Markus Scheidgen committed
208
209
    - if [ ${DEPLOYS}  -eq 0 ]; then
        helm install --name=${RELEASE_NAME} . --namespace=${STAGING_NAMESPACE}
210
          --set api.disableReset="false"
Markus Scheidgen's avatar
Markus Scheidgen committed
211
          --set proxy.nodePort="300${NUMERIC_VERSION//./}"
Markus Scheidgen's avatar
Markus Scheidgen committed
212
213
          --set proxy.external.path=${EXTERNAL_PATH}
          --set dbname=${DBNAME}
214
215
          --set worker.replicas=1
          --set worker.memrequest=32
Markus Scheidgen's avatar
Markus Scheidgen committed
216
217
          --set volumes.files=${FILES_PATH};
      else
218
        helm upgrade ${RELEASE_NAME} . --namespace=${STAGING_NAMESPACE} --recreate-pods;
Markus Scheidgen's avatar
Markus Scheidgen committed
219
      fi
Markus Scheidgen's avatar
Markus Scheidgen committed
220
221
  except:
    - /^dev-.*$/
222
  when: manual