.gitlab-ci.yml 7.13 KB
Newer Older
Markus Scheidgen's avatar
Markus Scheidgen committed
1
# default installed image for docker executor is: python:3.6
2
# using an image that can do git, docker, docker-compose
3
image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/ci-runner
4

5
# Uncomment the next lines, to run each pipeline/job in its own docker environment.
6
# Otherwise, it will use the docker of the gitlab runner host (e.g. enc-preprocessing...).
7
# This will give it access to a persistent layer cache, which will not be available
8
# with the docker service.
9 10
# services:
#   - docker:dind
Markus Scheidgen's avatar
Markus Scheidgen committed
11

12 13
# TODO remove CI_NOMAD_DEV_PROXY

Markus Scheidgen's avatar
Markus Scheidgen committed
14
stages:
Markus Scheidgen's avatar
Markus Scheidgen committed
15
  - build
16
  - test
17 18
  - release

19

20
variables:
21
  TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:${CI_COMMIT_REF_NAME}
Markus Scheidgen's avatar
Markus Scheidgen committed
22
  STABLE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:stable
Markus Scheidgen's avatar
Markus Scheidgen committed
23
  LATEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest
24 25
  KUBECONFIG: /etc/deploy/config

Markus Scheidgen's avatar
Markus Scheidgen committed
26 27
build:
  stage: build
28 29
  before_script:
    - git submodule sync
30
    - git submodule update --init --jobs=4
31
    # create the version information
32
    - ./gitinfo.sh
Markus Scheidgen's avatar
Markus Scheidgen committed
33
  script:
34 35
    # ignore test directories of dependencies, there is a lot of data that we not use
    # in the context of the docker image
36
    - find dependencies -name "*test*" -type d -exec echo "{}/" \;  >> .dockerignore
37
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
38
    - docker build -t $TEST_IMAGE .
39
    - docker push $TEST_IMAGE
40 41
  except:
    - /^dev-.*$/
Markus Scheidgen's avatar
Markus Scheidgen committed
42
    - tags
43

44
python linting:
45 46 47 48
  stage: test
  image: $TEST_IMAGE
  script:
    - cd /app
49
    - python -m pycodestyle --ignore=E501,E701,E731 nomad tests
50
    - python -m pylint --load-plugins=pylint_mongoengine,nomad.metainfo.pylint_plugin nomad tests
51 52 53 54 55 56 57 58
    - python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests
  except:
    refs:
      - tags
    variables:
      - $CI_COMMIT_REF_NAME =~ /^dev-.*$/
      - $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i

59
gui linting:
Markus Scheidgen's avatar
Markus Scheidgen committed
60 61 62 63 64
  stage: test
  image: node
  script:
    - cd gui
    - yarn
65
    - yarn run lint
66 67 68 69 70 71
  except:
    refs:
      - tags
    variables:
      - $CI_COMMIT_REF_NAME =~ /^dev-.*$/
      - $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
Markus Scheidgen's avatar
Markus Scheidgen committed
72

73
python tests:
Markus Scheidgen's avatar
Markus Scheidgen committed
74
  stage: test
Markus Scheidgen's avatar
Markus Scheidgen committed
75
  image: $TEST_IMAGE
76 77 78 79 80 81 82 83 84 85 86 87 88 89 90
  services:
    - rabbitmq
    - name: docker.elastic.co/elasticsearch/elasticsearch:6.3.2
      alias: elastic
      # fix issue with running elastic in gitlab ci runner:
      # https://gitlab.com/gitlab-org/gitlab-ce/issues/42214
      command: [ "bin/elasticsearch", "-Ediscovery.type=single-node" ]
  variables:
    RABBITMQ_ERLANG_COOKIE: SWQOKODSQALRPCLNMEQG
    RABBITMQ_DEFAULT_USER: rabbitmq
    RABBITMQ_DEFAULT_PASS: rabbitmq
    RABBITMQ_DEFAULT_VHOST: /
    NOMAD_RABBITMQ_HOST: rabbitmq
    NOMAD_ELASTIC_HOST: elastic
    NOMAD_MONGO_HOST: mongo
91
    NOMAD_KEYCLOAK_PASSWORD: ${CI_KEYCLOAK_ADMIN_PASSWORD}
92
    NOMAD_NORMALIZE_SPRINGER_DB_PATH: /nomad/fairdi/db/data/springer.msg
Markus Scheidgen's avatar
Markus Scheidgen committed
93
  script:
94
    - cd /app
95
    - ls /builds
96
    - python -m pytest --cov=nomad -sv tests
97
  except:
Markus Scheidgen's avatar
Markus Scheidgen committed
98 99
    refs:
      - tags
100 101
    variables:
      - $CI_COMMIT_REF_NAME =~ /^dev-.*$/
102
      - $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
103

104 105 106 107 108 109 110 111 112 113 114 115 116 117
gui tests:
  stage: test
  image: node
  script:
    - cd gui
    - yarn
    - yarn run test
  except:
    refs:
      - tags
    variables:
      - $CI_COMMIT_REF_NAME =~ /^dev-.*$/
      - $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i

118
install tests:
119
  stage: test
120
  image: python:3.7
121 122
  before_script:
    - git submodule sync
123
    - sleep 5
124 125 126 127 128 129 130 131
    - git submodule update --init --jobs=4
  script:
    - pip install --upgrade pip
    - pip install fastentrypoints
    - pip install pyyaml
    - pip install numpy
    - python setup.py compile
    - python setup.py sdist
132
    - pip install dist/nomad-lab-*.tar.gz
133
    - python -c "import nomad.datamodel, nomad.datamodel.metainfo, nomad.client"
134
    - pip install `echo dist/nomad-lab-*.tar.gz`[parsing]
135
    - python -m nomad.cli parse tests/data/parsers/vasp/vasp.xml
136 137 138 139 140 141
  except:
    refs:
      - tags
    variables:
      - $CI_COMMIT_REF_NAME =~ /^dev-.*$/
      - $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
142

143
dev setup tests:
144 145 146 147 148 149
  stage: test
  image: python:3.7
  script:
    - pip install --upgrade pip
    - ./setup.sh
    - python -m nomad.cli parse tests/data/parsers/vasp/vasp.xml
150 151
    - python -c "import nomad.app"
    - python -c "import nomad.processing"
152 153 154 155 156 157
  except:
    refs:
      - tags
    variables:
      - $CI_COMMIT_REF_NAME =~ /^dev-.*$/
      - $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
158

159 160
deploy dev:
  stage: release
161 162 163 164
  before_script:
    - mkdir -p /etc/deploy
    - echo ${CI_K8S_CONFIG} | base64 -d > ${KUBECONFIG}
  script:
165
    - RELEASE_NAME=`echo ${CI_COMMIT_REF_NAME} | sed -e 's/[^A-Za-z0-9\-]/-/g'`
166
    - helm dependency update ops/helm/nomad
167
    - helm upgrade --namespace nomad --install $RELEASE_NAME ops/helm/nomad -f ops/helm/nomad/deployments/dev-values.yaml --set proxy.external.path=/dev/nomad/$RELEASE_NAME,image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
168
    - docker pull $TEST_IMAGE
169 170 171 172
    - docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n https://nomad-lab.eu/dev/nomad/$RELEASE_NAME/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests  --skip-publish --skip-doi
  except:
    - /^dev-.*$/
  when: manual
173

174 175 176 177 178 179 180 181 182 183
deploy prod beta:
  stage: release
  before_script:
    - mkdir -p /etc/deploy
    - echo ${CI_K8S_PROD_CONFIG} | base64 -d > ${KUBECONFIG}
  script:
    - helm dependency update ops/helm/nomad
    - helm upgrade --install nomad-prod-test ops/helm/nomad -f ops/helm/nomad/deployments/prod-beta-values.yaml --set image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
    - docker pull $TEST_IMAGE
    - docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n https://nomad-lab.eu/prod/rae/beta/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests  --skip-publish --skip-doi
184 185 186 187
  except:
    - /^dev-.*$/
  when: manual

188
release latest image:
189 190 191
  stage: release
  script:
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
Markus Scheidgen's avatar
Markus Scheidgen committed
192
    - docker pull $TEST_IMAGE
Markus Scheidgen's avatar
Markus Scheidgen committed
193 194 195 196
    - docker tag $TEST_IMAGE $LATEST_IMAGE
    - docker push $LATEST_IMAGE
  except:
    - /^dev-.*$/
Markus Scheidgen's avatar
Markus Scheidgen committed
197
  when: manual
Markus Scheidgen's avatar
Markus Scheidgen committed
198

199
release stable image:
Markus Scheidgen's avatar
Markus Scheidgen committed
200 201 202
  stage: release
  script:
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
203 204 205 206
    - docker pull $TEST_IMAGE
    - docker tag $TEST_IMAGE $LATEST_IMAGE
    - docker push $LATEST_IMAGE
    - docker tag $TEST_IMAGE $STABLE_IMAGE
Markus Scheidgen's avatar
Markus Scheidgen committed
207
    - docker push $STABLE_IMAGE
208
  when: manual
Markus Scheidgen's avatar
Markus Scheidgen committed
209 210
  only:
    - tags
211

212
python package:
213 214
  stage: release
  script:
215 216 217 218 219 220 221 222 223 224 225 226
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
    - docker pull $TEST_IMAGE
    - docker run --rm $TEST_IMAGE python -m twine upload -u gitlab-ci-token -p ${CI_JOB_TOKEN} --repository-url https://gitlab.mpcdf.mpg.de/api/v4/projects/${CI_PROJECT_ID}/packages/pypi dist/nomad-lab.tar.gz
  except:
    - /^dev-.*$/
  when: manual

pypi package:
  stage: release
  script:
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
    - docker pull $TEST_IMAGE
227
    - docker run --rm $TEST_IMAGE bash -c "python -m twine upload -u $CI_TWINE_USER -p $CI_TWINE_PASSWORD dist/nomad-lab-*.tar.gz"
228 229 230
  when: manual
  only:
    - tags