Commit 01f31b8b authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added python package release to gitlab pypi registry. Refactored deploy and release.

parent c72c9ccc
Pipeline #89724 passed with stages
in 31 minutes
......@@ -9,10 +9,11 @@ image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/ci-runner
# services:
# - docker:dind
# TODO remove CI_NOMAD_DEV_PROXY
stages:
- build
- test
- deploy
- release
......@@ -40,7 +41,7 @@ build:
- /^dev-.*$/
- tags
linting:
python linting:
stage: test
image: $TEST_IMAGE
script:
......@@ -55,7 +56,7 @@ linting:
- $CI_COMMIT_REF_NAME =~ /^dev-.*$/
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
gui_linting:
gui linting:
stage: test
image: node
script:
......@@ -100,7 +101,7 @@ tests:
- $CI_COMMIT_REF_NAME =~ /^dev-.*$/
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
install_tests:
install tests:
stage: test
image: python:3.7
before_script:
......@@ -124,7 +125,7 @@ install_tests:
- $CI_COMMIT_REF_NAME =~ /^dev-.*$/
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
dev_setup_tests:
dev setup tests:
stage: test
image: python:3.7
script:
......@@ -138,23 +139,36 @@ dev_setup_tests:
- $CI_COMMIT_REF_NAME =~ /^dev-.*$/
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
deploy:
stage: deploy
deploy dev:
stage: release
before_script:
- mkdir -p /etc/deploy
- echo ${CI_K8S_CONFIG} | base64 -d > ${KUBECONFIG}
script:
- RELEASE_NAME=`echo ${CI_COMMIT_REF_NAME} | sed -e 's/[^A-Za-z0-9\-]/-/g'`
- helm dependency update ops/helm/nomad
- helm upgrade --namespace nomad --install $RELEASE_NAME ops/helm/nomad -f ops/helm/nomad/ci-dev-values.yaml --set proxy.external.path=/dev/nomad/$RELEASE_NAME,image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
- helm upgrade --namespace nomad --install $RELEASE_NAME ops/helm/nomad -f ops/helm/nomad/deployments/dev-values.yaml --set proxy.external.path=/dev/nomad/$RELEASE_NAME,image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
- docker pull $TEST_IMAGE
- docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n $CI_DEV_CLUSTER_PROXY/dev/nomad/$RELEASE_NAME/api -u admin -w $CI_NOMAD_ADMIN_PASSWORD integrationtests
- docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n https://nomad-lab.eu/dev/nomad/$RELEASE_NAME/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests --skip-publish --skip-doi
except:
- /^dev-.*$/
when: manual
deploy prod beta:
stage: release
before_script:
- mkdir -p /etc/deploy
- echo ${CI_K8S_PROD_CONFIG} | base64 -d > ${KUBECONFIG}
script:
- helm dependency update ops/helm/nomad
- helm upgrade --install nomad-prod-test ops/helm/nomad -f ops/helm/nomad/deployments/prod-beta-values.yaml --set image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
- docker pull $TEST_IMAGE
- docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n https://nomad-lab.eu/prod/rae/beta/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests --skip-publish --skip-doi
except:
- /^dev-.*$/
when: manual
release:
release latest image:
stage: release
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
......@@ -164,10 +178,8 @@ release:
except:
- /^dev-.*$/
when: manual
only:
- branches
release_version:
release stable image:
stage: release
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
......@@ -176,24 +188,26 @@ release_version:
- docker push $LATEST_IMAGE
- docker tag $TEST_IMAGE $STABLE_IMAGE
- docker push $STABLE_IMAGE
when: manual
only:
- tags
release_pypi:
python package:
stage: release
image: python:3.7
before_script:
- git submodule sync
- git submodule update --init --jobs=4
script:
- pip install --upgrade pip
- pip install fastentrypoints
- pip install pyyaml
- pip install numpy
- pip install twine
- python setup.py compile
- python setup.py sdist
- python -m twine upload -u $CI_TWINE_USER -p $CI_TWINE_PASSWORD dist/*
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker pull $TEST_IMAGE
- docker run --rm $TEST_IMAGE python -m twine upload -u gitlab-ci-token -p ${CI_JOB_TOKEN} --repository-url https://gitlab.mpcdf.mpg.de/api/v4/projects/${CI_PROJECT_ID}/packages/pypi dist/nomad-lab.tar.gz
except:
- /^dev-.*$/
when: manual
pypi package:
stage: release
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker pull $TEST_IMAGE
- docker run --rm $TEST_IMAGE python -m twine upload -u $CI_TWINE_USER -p $CI_TWINE_PASSWORD dist/nomad-lab.tar.gz
when: manual
only:
- tags
......@@ -3,7 +3,7 @@ image:
proxy:
external:
host: "labdev-nomad.esc.rzg.mpg.de"
host: "nomad-lab.eu"
ingress:
enabled: true
......@@ -28,6 +28,7 @@ dbname: nomad_dev_v0_8
keycloak:
serverUrl: "https://nomad-lab.eu/fairdi/keycloak/auth/"
serverExternalUrl: "https://nomad-lab.eu/fairdi/keycloak/auth/"
passwordSecret: 'nomad-keycloak-password'
realmName: 'fairdi_nomad_prod'
clientId: 'nomad_public'
......
version:
label: "v0.9.8"
isBeta: true
usesBetaData: false
officialUrl: "https://nomad-lab.eu/prod/rae/gui"
image:
tag: "latest"
pullPolicy: "Always"
proxy:
nodePort: 30014
nodeIP: '130.183.207.81'
external:
host: "nomad-lab.eu"
path: "/prod/rae/beta"
gui:
debug: true
matomoEnabled: false
matomoSiteId: 2
encyclopediaEnabled: true
aitoolkitEnabled: true
worker:
replicas: 1
routing: "queue"
processes: 10
nomadNodeType: "prod-worker"
elastic:
host: elasticsearch.elasticsearch.svc.cluster.local
port: 9200
mongo:
host: rs0/mongodb-0.mongo.mongodb.svc.cluster.local,mongodb-1.mongo.mongodb.svc.cluster.local,mongodb-2.mongo.mongodb.svc.cluster.local
logstash:
host: logstash.elk.svc.cluster.local
dbname: fairdi_nomad_prod_v0_8
uploadurl: 'http://nomad-lab.eu/prod/rae/beta/api/uploads'
client:
passwordSecret: 'nomad-keycloak-password'
keycloak:
serverUrl: "https://nomad-lab.eu/fairdi/keycloak/auth/"
serverExternalUrl: "https://nomad-lab.eu/fairdi/keycloak/auth/"
passwordSecret: 'nomad-keycloak-password'
realmName: 'fairdi_nomad_prod'
clientId: 'nomad_public'
admin_user_id: '82efac55-6187-408c-8027-b98580c0e1c5'
volumes:
prefixSize: 1
public: /nomad/fairdi/prod/fs/public
staging: /nomad/fairdi/prod/fs/staging
tmp: /nomad/fairdi/prod/fs/tmp
nomad: /nomad
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment