Added python package release to gitlab pypi registry. Refactored deploy and release.

.gitlab-ci.yml

@@ -9,10 +9,11 @@ image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/ci-runner
 # services:
 #   - docker:dind
 
+# TODO remove CI_NOMAD_DEV_PROXY
+
 stages:
   - build
   - test
-  - deploy
   - release
 
 
@@ -40,7 +41,7 @@ build:
     - /^dev-.*$/
     - tags
 
-linting:
+python linting:
   stage: test
   image: $TEST_IMAGE
   script:
@@ -55,7 +56,7 @@ linting:
       - $CI_COMMIT_REF_NAME =~ /^dev-.*$/
       - $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
 
-gui_linting:
+gui linting:
   stage: test
   image: node
   script:
@@ -100,7 +101,7 @@ tests:
       - $CI_COMMIT_REF_NAME =~ /^dev-.*$/
       - $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
 
-install_tests:
+install tests:
   stage: test
   image: python:3.7
   before_script:
@@ -124,7 +125,7 @@ install_tests:
       - $CI_COMMIT_REF_NAME =~ /^dev-.*$/
       - $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
 
-dev_setup_tests:
+dev setup tests:
   stage: test
   image: python:3.7
   script:
@@ -138,23 +139,36 @@ dev_setup_tests:
       - $CI_COMMIT_REF_NAME =~ /^dev-.*$/
       - $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
 
-deploy:
-  stage: deploy
+deploy dev:
+  stage: release
   before_script:
     - mkdir -p /etc/deploy
     - echo ${CI_K8S_CONFIG} | base64 -d > ${KUBECONFIG}
   script:
     - RELEASE_NAME=`echo ${CI_COMMIT_REF_NAME} | sed -e 's/[^A-Za-z0-9\-]/-/g'`
     - helm dependency update ops/helm/nomad
-    - helm upgrade --namespace nomad --install $RELEASE_NAME ops/helm/nomad -f ops/helm/nomad/ci-dev-values.yaml --set proxy.external.path=/dev/nomad/$RELEASE_NAME,image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
+    - helm upgrade --namespace nomad --install $RELEASE_NAME ops/helm/nomad -f ops/helm/nomad/deployments/dev-values.yaml --set proxy.external.path=/dev/nomad/$RELEASE_NAME,image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
     - docker pull $TEST_IMAGE
-    - docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n $CI_DEV_CLUSTER_PROXY/dev/nomad/$RELEASE_NAME/api -u admin -w $CI_NOMAD_ADMIN_PASSWORD integrationtests
+    - docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n https://nomad-lab.eu/dev/nomad/$RELEASE_NAME/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests  --skip-publish --skip-doi
+  except:
+    - /^dev-.*$/
+  when: manual
 
+deploy prod beta:
+  stage: release
+  before_script:
+    - mkdir -p /etc/deploy
+    - echo ${CI_K8S_PROD_CONFIG} | base64 -d > ${KUBECONFIG}
+  script:
+    - helm dependency update ops/helm/nomad
+    - helm upgrade --install nomad-prod-test ops/helm/nomad -f ops/helm/nomad/deployments/prod-beta-values.yaml --set image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
+    - docker pull $TEST_IMAGE
+    - docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n https://nomad-lab.eu/prod/rae/beta/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests  --skip-publish --skip-doi
   except:
     - /^dev-.*$/
   when: manual
 
-release:
+release latest image:
   stage: release
   script:
     - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
@@ -164,10 +178,8 @@ release:
   except:
     - /^dev-.*$/
   when: manual
-  only:
-    - branches
 
-release_version:
+release stable image:
   stage: release
   script:
     - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
@@ -176,24 +188,26 @@ release_version:
     - docker push $LATEST_IMAGE
     - docker tag $TEST_IMAGE $STABLE_IMAGE
     - docker push $STABLE_IMAGE
+  when: manual
   only:
     - tags
 
-release_pypi:
+python package:
   stage: release
-  image: python:3.7
-  before_script:
-    - git submodule sync
-    - git submodule update --init --jobs=4
   script:
-    - pip install --upgrade pip
-    - pip install fastentrypoints
-    - pip install pyyaml
-    - pip install numpy
-    - pip install twine
-    - python setup.py compile
-    - python setup.py sdist
-    - python -m twine upload -u $CI_TWINE_USER -p $CI_TWINE_PASSWORD dist/*
+    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
+    - docker pull $TEST_IMAGE
+    - docker run --rm $TEST_IMAGE python -m twine upload -u gitlab-ci-token -p ${CI_JOB_TOKEN} --repository-url https://gitlab.mpcdf.mpg.de/api/v4/projects/${CI_PROJECT_ID}/packages/pypi dist/nomad-lab.tar.gz
+  except:
+    - /^dev-.*$/
+  when: manual
+
+pypi package:
+  stage: release
+  script:
+    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
+    - docker pull $TEST_IMAGE
+    - docker run --rm $TEST_IMAGE python -m twine upload -u $CI_TWINE_USER -p $CI_TWINE_PASSWORD dist/nomad-lab.tar.gz
   when: manual
   only:
     - tags

ops/.gitignore

-deployments

ops/helm/nomad/ci-dev-values.yaml → ops/helm/nomad/deployments/dev-values.yaml

@@ -3,7 +3,7 @@ image:
 
 proxy:
   external:
-    host: "labdev-nomad.esc.rzg.mpg.de"
+    host: "nomad-lab.eu"
 
 ingress:
   enabled: true
@@ -28,6 +28,7 @@ dbname: nomad_dev_v0_8
 
 keycloak:
   serverUrl: "https://nomad-lab.eu/fairdi/keycloak/auth/"
+  serverExternalUrl: "https://nomad-lab.eu/fairdi/keycloak/auth/"
   passwordSecret: 'nomad-keycloak-password'
   realmName: 'fairdi_nomad_prod'
   clientId: 'nomad_public'

ops/helm/nomad/deployments/prod-beta-values.yaml

+version:
+  label: "v0.9.8"
+  isBeta: true
+  usesBetaData: false
+  officialUrl: "https://nomad-lab.eu/prod/rae/gui"
+
+image:
+    tag: "latest"
+    pullPolicy: "Always"
+
+proxy:
+  nodePort: 30014
+  nodeIP: '130.183.207.81'
+  external:
+    host: "nomad-lab.eu"
+    path: "/prod/rae/beta"
+
+gui:
+  debug: true
+  matomoEnabled: false
+  matomoSiteId: 2
+  encyclopediaEnabled: true
+  aitoolkitEnabled: true
+
+worker:
+  replicas: 1
+  routing: "queue"
+  processes: 10
+  nomadNodeType: "prod-worker"
+
+elastic:
+  host: elasticsearch.elasticsearch.svc.cluster.local
+  port: 9200
+
+mongo:
+  host: rs0/mongodb-0.mongo.mongodb.svc.cluster.local,mongodb-1.mongo.mongodb.svc.cluster.local,mongodb-2.mongo.mongodb.svc.cluster.local
+
+logstash:
+  host: logstash.elk.svc.cluster.local
+
+dbname: fairdi_nomad_prod_v0_8
+
+uploadurl: 'http://nomad-lab.eu/prod/rae/beta/api/uploads'
+
+client:
+  passwordSecret: 'nomad-keycloak-password'
+
+keycloak:
+  serverUrl: "https://nomad-lab.eu/fairdi/keycloak/auth/"
+  serverExternalUrl: "https://nomad-lab.eu/fairdi/keycloak/auth/"
+  passwordSecret: 'nomad-keycloak-password'
+  realmName: 'fairdi_nomad_prod'
+  clientId: 'nomad_public'
+  admin_user_id: '82efac55-6187-408c-8027-b98580c0e1c5'
+
+volumes:
+  prefixSize: 1
+  public: /nomad/fairdi/prod/fs/public
+  staging: /nomad/fairdi/prod/fs/staging
+  tmp: /nomad/fairdi/prod/fs/tmp
+  nomad: /nomad

requirements.txt

@@ -94,3 +94,4 @@ rope
 mongomock==3.17.0
 names
 essential_generators
+twine