.gitlab-ci.yml 5.21 KB
Newer Older
Markus Scheidgen's avatar
Markus Scheidgen committed
1
# default installed image for docker executor is: python:3.6
2
# using an image that can do git, docker, docker-compose
3
image: youpy/docker-compose-git
4
5
6
7
8

# Uncomment the next lines, to run each pipline/job in its own docker environment.
# Otherwise, it will use the docker of the gitlab runner host (e.g. enc-preprocessing...).
# This will give it access to a persitent layer cache, which will not be available
# with the docker service.
9
10
# services:
#   - docker:dind
Markus Scheidgen's avatar
Markus Scheidgen committed
11
12

stages:
Markus Scheidgen's avatar
Markus Scheidgen committed
13
  - build
14
  - test
15
  - integration
16
  - release
17
  - deploy
18

19
variables:
Markus Scheidgen's avatar
Markus Scheidgen committed
20
21
  TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:test_${CI_COMMIT_REF_NAME}
  RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:${CI_COMMIT_REF_NAME}
Markus Scheidgen's avatar
Markus Scheidgen committed
22
  STABLE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:stable
Markus Scheidgen's avatar
Markus Scheidgen committed
23
24
25
  LATEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest
  FRONTEND_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:test_${CI_COMMIT_REF_NAME}
  FRONTEND_RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:${CI_COMMIT_REF_NAME}
Markus Scheidgen's avatar
Markus Scheidgen committed
26
  FRONTEND_STABLE_IMAGE:  gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:stable
Markus Scheidgen's avatar
Markus Scheidgen committed
27
  FRONTEND_LATEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:latest
28
29
  KUBECONFIG: /etc/deploy/config

Markus Scheidgen's avatar
Markus Scheidgen committed
30
31
build:
  stage: build
32
33
  before_script:
    - git submodule sync
34
    - git submodule update --init --jobs=4
35
    - ./gitinfo.sh
Markus Scheidgen's avatar
Markus Scheidgen committed
36
  script:
37
38
    # ignore test directories of dependencies, there is a lot of data that we not use
    # in the context of the docker image
39
    - find dependencies -name "*test*" -type d -exec echo "{}/" \;  >> .dockerignore
40
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
41
    - docker build -t $TEST_IMAGE .
Markus Scheidgen's avatar
Markus Scheidgen committed
42
    - docker push $TEST_IMAGE
43
44
  except:
    - /^dev-.*$/
Markus Scheidgen's avatar
Markus Scheidgen committed
45
    - tags
46

Markus Scheidgen's avatar
Markus Scheidgen committed
47
48
49
50
51

buildgui:
  stage: build
  script:
    - cd gui
52
    - ./version.sh
Markus Scheidgen's avatar
Markus Scheidgen committed
53
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
54
    - docker build -t $FRONTEND_TEST_IMAGE .
Markus Scheidgen's avatar
Markus Scheidgen committed
55
    - docker push $FRONTEND_TEST_IMAGE
56
57
  except:
    - /^dev-.*$/
Markus Scheidgen's avatar
Markus Scheidgen committed
58
    - tags
59

60
61
62
63
64
linting:
  stage: test
  image: $TEST_IMAGE
  script:
    - cd /app
65
    - python -m pycodestyle --ignore=E501,E701,E731 nomad tests
66
67
68
69
70
71
72
73
74
    - python -m pylint --load-plugins=pylint_mongoengine nomad tests
    - python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests
  except:
    refs:
      - tags
    variables:
      - $CI_COMMIT_REF_NAME =~ /^dev-.*$/
      - $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i

75
tests:
Markus Scheidgen's avatar
Markus Scheidgen committed
76
  stage: test
Markus Scheidgen's avatar
Markus Scheidgen committed
77
  image: $TEST_IMAGE
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
  services:
    - rabbitmq
    - name: docker.elastic.co/elasticsearch/elasticsearch:6.3.2
      alias: elastic
      # fix issue with running elastic in gitlab ci runner:
      # https://gitlab.com/gitlab-org/gitlab-ce/issues/42214
      command: [ "bin/elasticsearch", "-Ediscovery.type=single-node" ]
  variables:
    RABBITMQ_ERLANG_COOKIE: SWQOKODSQALRPCLNMEQG
    RABBITMQ_DEFAULT_USER: rabbitmq
    RABBITMQ_DEFAULT_PASS: rabbitmq
    RABBITMQ_DEFAULT_VHOST: /
    NOMAD_RABBITMQ_HOST: rabbitmq
    NOMAD_ELASTIC_HOST: elastic
    NOMAD_MONGO_HOST: mongo
93
    NOMAD_KEYCLOAK_CLIENT_SECRET: ${CI_KEYCLOAK_TEST_CLIENT_SECRET}
94
    NOMAD_KEYCLOAK_PASSWORD: ${CI_KEYCLOAK_ADMIN_PASSWORD}
95
    NOMAD_SPRINGER_DB_PATH: /nomad/fairdi/db/data/springer.db
Markus Scheidgen's avatar
Markus Scheidgen committed
96
  script:
97
    - cd /app
98
    - ls /builds
99
    - python -m pytest --cov=nomad -sv tests
100
  except:
Markus Scheidgen's avatar
Markus Scheidgen committed
101
102
    refs:
      - tags
103
104
    variables:
      - $CI_COMMIT_REF_NAME =~ /^dev-.*$/
105
      - $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
106

107
108
109
110
release:
  stage: release
  script:
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
Markus Scheidgen's avatar
Markus Scheidgen committed
111
    - docker pull $TEST_IMAGE
Markus Scheidgen's avatar
Markus Scheidgen committed
112
113
114
115
116
117
118
119
120
121
122
123
124
    - docker tag $TEST_IMAGE $LATEST_IMAGE
    - docker push $LATEST_IMAGE
    - docker pull $FRONTEND_TEST_IMAGE
    - docker tag $FRONTEND_TEST_IMAGE $FRONTEND_LATEST_IMAGE
    - docker push $FRONTEND_LATEST_IMAGE
  except:
    - /^dev-.*$/
  when: manual

release_version:
  stage: release
  script:
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
Markus Scheidgen's avatar
Markus Scheidgen committed
125
126
    - docker pull $LATEST_IMAGE
    - docker tag $LATEST_IMAGE $RELEASE_IMAGE
Markus Scheidgen's avatar
Markus Scheidgen committed
127
    - docker tag $LATEST_IMAGE $STABLE_IMAGE
Markus Scheidgen's avatar
Markus Scheidgen committed
128
    - docker push $RELEASE_IMAGE
Markus Scheidgen's avatar
Markus Scheidgen committed
129
    - docker push $STABLE_IMAGE
Markus Scheidgen's avatar
Markus Scheidgen committed
130
131
    - docker pull $FRONTEND_LATEST_IMAGE
    - docker tag $FRONTEND_LATEST_IMAGE $FRONTEND_RELEASE_IMAGE
Markus Scheidgen's avatar
Markus Scheidgen committed
132
    - docker tag $FRONTEND_LATEST_IMAGE $FRONTEND_STABLE_IMAGE
Markus Scheidgen's avatar
Markus Scheidgen committed
133
    - docker push $FRONTEND_RELEASE_IMAGE
Markus Scheidgen's avatar
Markus Scheidgen committed
134
    - docker push $FRONTEND_STABLE_IMAGE
Markus Scheidgen's avatar
Markus Scheidgen committed
135
136
  only:
    - tags
137

138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
## TODO This has to be fixed, we need one for testing, staging, production
# deploy_coe_staging:
#   stage: deploy
#   image: dtzar/helm-kubectl
#   before_script:
#     - mkdir -p /etc/deploy
#     # kube_config is a CI/CD variable set in GitLab GUI
#     - echo $CI_KUBE_CONFIG | base64 -d > /etc/deploy/config
#     - helm init --upgrade
#     - helm repo add stable https://kubernetes-charts.storage.googleapis.com/
#     - helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com/
#     - helm repo update
#   script:
#     - cd ops/helm/nomad
#     - export KUBECONFIG=/etc/deploy/config
#     - helm dep build
#     - helm upgrade nomad-coe-staging . --recreate-pods;
#   except:
#     - /^dev-.*$/
#   when: manual