.gitlab-ci.yml 4.06 KB
Newer Older
Markus Scheidgen's avatar
Markus Scheidgen committed
1
# default installed image for docker executor is: python:3.6
2
# using an image that can do git, docker, docker-compose
3
image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/ci-runner
4
5
6
7
8

# Uncomment the next lines, to run each pipline/job in its own docker environment.
# Otherwise, it will use the docker of the gitlab runner host (e.g. enc-preprocessing...).
# This will give it access to a persitent layer cache, which will not be available
# with the docker service.
9
10
# services:
#   - docker:dind
Markus Scheidgen's avatar
Markus Scheidgen committed
11
12

stages:
Markus Scheidgen's avatar
Markus Scheidgen committed
13
  - build
14
  - test
15
  - deploy
16
17
  - release

18

19
variables:
20
  TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:${CI_COMMIT_REF_NAME}
Markus Scheidgen's avatar
Markus Scheidgen committed
21
  STABLE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:stable
Markus Scheidgen's avatar
Markus Scheidgen committed
22
  LATEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest
23
24
  KUBECONFIG: /etc/deploy/config

Markus Scheidgen's avatar
Markus Scheidgen committed
25
26
build:
  stage: build
27
28
  before_script:
    - git submodule sync
29
    - git submodule update --init
30
    # create the version information
31
    - ./gitinfo.sh
Markus Scheidgen's avatar
Markus Scheidgen committed
32
  script:
33
34
    # ignore test directories of dependencies, there is a lot of data that we not use
    # in the context of the docker image
35
    - find dependencies -name "*test*" -type d -exec echo "{}/" \;  >> .dockerignore
36
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
37
    - docker build -t $TEST_IMAGE .
Markus Scheidgen's avatar
Markus Scheidgen committed
38
    - docker push $TEST_IMAGE
39
40
  except:
    - /^dev-.*$/
Markus Scheidgen's avatar
Markus Scheidgen committed
41
    - tags
42

43
44
45
46
47
linting:
  stage: test
  image: $TEST_IMAGE
  script:
    - cd /app
48
    - python -m pycodestyle --ignore=E501,E701,E731 nomad tests
49
    - python -m pylint --load-plugins=pylint_mongoengine,nomad/metainfo/pylint_plugin nomad tests
50
51
52
53
54
55
56
57
    - python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests
  except:
    refs:
      - tags
    variables:
      - $CI_COMMIT_REF_NAME =~ /^dev-.*$/
      - $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i

58
tests:
Markus Scheidgen's avatar
Markus Scheidgen committed
59
  stage: test
Markus Scheidgen's avatar
Markus Scheidgen committed
60
  image: $TEST_IMAGE
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
  services:
    - rabbitmq
    - name: docker.elastic.co/elasticsearch/elasticsearch:6.3.2
      alias: elastic
      # fix issue with running elastic in gitlab ci runner:
      # https://gitlab.com/gitlab-org/gitlab-ce/issues/42214
      command: [ "bin/elasticsearch", "-Ediscovery.type=single-node" ]
  variables:
    RABBITMQ_ERLANG_COOKIE: SWQOKODSQALRPCLNMEQG
    RABBITMQ_DEFAULT_USER: rabbitmq
    RABBITMQ_DEFAULT_PASS: rabbitmq
    RABBITMQ_DEFAULT_VHOST: /
    NOMAD_RABBITMQ_HOST: rabbitmq
    NOMAD_ELASTIC_HOST: elastic
    NOMAD_MONGO_HOST: mongo
76
    NOMAD_KEYCLOAK_PASSWORD: ${CI_KEYCLOAK_ADMIN_PASSWORD}
77
    NOMAD_NORMALIZE_SPRINGER_DB_PATH: /nomad/fairdi/db/data/springer.msg
Markus Scheidgen's avatar
Markus Scheidgen committed
78
  script:
79
    - cd /app
80
    - ls /builds
Markus Scheidgen's avatar
Markus Scheidgen committed
81
    - python -m pytest --cov=nomad -sv tests
82
  except:
Markus Scheidgen's avatar
Markus Scheidgen committed
83
84
    refs:
      - tags
85
86
    variables:
      - $CI_COMMIT_REF_NAME =~ /^dev-.*$/
87
      - $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
88

89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
deploy:
  stage: deploy
  before_script:
    - mkdir -p /etc/deploy
    - echo ${CI_K8S_CONFIG} | base64 -d > ${KUBECONFIG}
  script:
    - helm dependency update ops/helm/nomad
    - helm upgrade --namespace nomad --install $CI_COMMIT_REF_NAME ops/helm/nomad -f ops/helm/nomad/ci-dev-values.yaml --set proxy.external.path=/dev/nomad/$CI_COMMIT_REF_NAME,image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
    - docker pull $TEST_IMAGE
    - docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n $CI_DEV_CLUSTER_PROXY/dev/nomad/$CI_COMMIT_REF_NAME/api -u admin -w $CI_NOMAD_ADMIN_PASSWORD integrationtests --skip-doi

  except:
    - /^dev-.*$/
  when: manual

104
105
106
107
release:
  stage: release
  script:
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
Markus Scheidgen's avatar
Markus Scheidgen committed
108
    - docker pull $TEST_IMAGE
Markus Scheidgen's avatar
Markus Scheidgen committed
109
110
    - docker tag $TEST_IMAGE $LATEST_IMAGE
    - docker push $LATEST_IMAGE
111

Markus Scheidgen's avatar
Markus Scheidgen committed
112
113
114
115
116
117
118
119
  except:
    - /^dev-.*$/
  when: manual

release_version:
  stage: release
  script:
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
Markus Scheidgen's avatar
Markus Scheidgen committed
120
    - docker pull $LATEST_IMAGE
121
    - docker tag $LATEST_IMAGE $CI_COMMIT_TAG
Markus Scheidgen's avatar
Markus Scheidgen committed
122
123
    - docker tag $LATEST_IMAGE $STABLE_IMAGE
    - docker push $STABLE_IMAGE
124
    - docker push $CI_COMMIT_TAG
Markus Scheidgen's avatar
Markus Scheidgen committed
125
126
  only:
    - tags