Merge branch 'migration' of gitlab.mpcdf.mpg.de:nomad-lab/nomad-FAIR into migration

.gitlab-ci.yml

@@ -17,13 +17,16 @@ stages:
   - deploy
 
 variables:
-  TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:test
+  TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:${CI_COMMIT_REF_NAME}
   RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest
-  FRONTEND_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:test
+  FRONTEND_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:${CI_COMMIT_REF_NAME}
   FRONTEND_RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:latest
-  RAWAPI_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/rawapi:test
+  RAWAPI_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/rawapi:${CI_COMMIT_REF_NAME}
   RAWAPI_RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/rawapi:latest
 
+  KUBECONFIG: /etc/deploy/config
+  STAGING_NAMESPACE: nomad-fairdi
+
 build:
   stage: build
   script:
@@ -53,8 +56,8 @@ buildrawapi:
     - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
     - docker build -t $RAWAPI_TEST_IMAGE -f rawapi.Dockerfile .
     - docker push $RAWAPI_TEST_IMAGE
-  except:
-    - /^dev-.*$/
+  only:
+    - rawapi
 
 
 linting:
@@ -160,22 +163,8 @@ release_rawapi:
     - docker pull $RAWAPI_TEST_IMAGE
     - docker tag $RAWAPI_TEST_IMAGE $RAWAPI_RELEASE_IMAGE
     - docker push $RAWAPI_RELEASE_IMAGE
-  except:
-    - /^dev-.*$/
-  when: manual
-
-deploy:
-  stage: deploy
-  script:
-    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
-    - cd ops/docker-compose/nomad
-    # secret .env file is stored on the "production" machine and is bind mounted into the
-    # runner container at /nomad/config/.env
-    - cp /nomad/config/.env .
-    - docker-compose -f docker-compose.yml -f docker-compose.prod.yml build
-    - docker-compose -f docker-compose.yml -f docker-compose.prod.yml up --no-deps -d api worker gui
-  except:
-    - /^dev-.*$/
+  only:
+    - rawapi
   when: manual
 
 deploy_rawapi:
@@ -187,6 +176,41 @@ deploy_rawapi:
     # runner container at /nomad/config/.env
     - cp /nomad/config/.rawapi_env .env
     - docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d
+  only:
+    - rawapi
+  when: manual
+
+deploy:
+  stage: deploy
+  image: dtzar/helm-kubectl
+  before_script:
+    - mkdir -p /etc/deploy
+    # kube_config is a CI/CD variable set in GitLab GUI
+    - echo $CI_KUBE_CONFIG | base64 -d > /etc/deploy/config
+    - helm init --client-only
+    - helm repo add stable https://kubernetes-charts.storage.googleapis.com/
+    - helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com/
+    - helm repo update
+  script:
+    - cd ops/helm/nomad
+    - export KUBECONFIG=/etc/deploy/config
+    - helm dep build
+    - export NOMAD_VERSION="$(grep "version" Chart.yaml | cut -d" " -f2)"
+    - export NUMERIC_VERSION="$(echo ${NOMAD_VERSION} | cut -d"." -f1-2)"
+    - export RELEASE_NAME="nomad-v${NOMAD_VERSION//./-}"
+    - export DEPLOYS=$(helm ls | grep $RELEASE_NAME | wc -l)
+    - export EXTERNAL_PATH="/fairdi/nomad/v${NOMAD_VERSION}"
+    - export DBNAME="fairdi_nomad_v${NOMAD_VERSION//./_}"
+    - export FILES_PATH="/scratch/nomad-fair/fs/nomad_v${NOMAD_VERSION}"
+    - if [ ${DEPLOYS}  -eq 0 ]; then
+        helm install --name=${RELEASE_NAME} . --namespace=${STAGING_NAMESPACE}
+          --set proxy.nodePort="300${NUMERIC_VERSION//./}"
+          --set proxy.external.path=${EXTERNAL_PATH}
+          --set dbname=${DBNAME}
+          --set volumes.files=${FILES_PATH};
+      else
+        helm upgrade ${RELEASE_NAME} . --namespace=${STAGING_NAMESPACE};
+      fi
   except:
     - /^dev-.*$/
   when: manual

gui/.env.development

 REACT_APP_API_BASE = 'http://localhost:8000/nomad/api'
 REACT_APP_APP_BASE = '/nomad'
 REACT_APP_APP_STATIC_BASE = ''
-REACT_APP_DEBUG = 'true'
\ No newline at end of file
+REACT_APP_DEBUG = 'true'
+REACT_KIBANA_BASE = '/nomad/kibana'
\ No newline at end of file

gui/src/components/Development.js

@@ -3,7 +3,7 @@ import PropTypes from 'prop-types'
 import { withStyles } from '@material-ui/core/styles'
 import Markdown from './Markdown'
 import gitInfo from '../gitinfo'
-import { appBase } from '../config'
+import { kibanaBase } from '../config'
 
 class Development extends React.Component {
   static propTypes = {
@@ -31,7 +31,7 @@ class Development extends React.Component {
           This system pushes logs, events, monitoring data,
           and other application metrics to a central database where it
           can be analysed visually.
-          \n\n[Link to Kiaba](${appBase}/kibana/)
+          \n\n[Link to Kiaba](${kibanaBase}/)
         `}</Markdown>
       </div>
     )

gui/src/config.js

@@ -8,6 +8,7 @@ import { createMuiTheme } from '@material-ui/core'
 window.nomadEnv = window.nomadEnv || {}
 export const apiBase = process.env.REACT_APP_API_BASE || window.nomadEnv.apiBase
 export const appBase = process.env.REACT_APP_APP_BASE || window.nomadEnv.appBase
+export const kibanaBase = process.env.REACT_KIBANA_BASE || window.nomadEnv.kibanaBase
 export const appStaticBase = process.env.REACT_APP_APP_STATIC_BASE || window.nomadEnv.appStaticBase
 export const debug = process.env.REACT_APP_DEBUG ? process.env.REACT_APP_DEBUG === 'true' : window.nomadEnv.debug
 

ops/helm/nomad/Chart.yaml

 apiVersion: v1
-appVersion: "1.0"
+appVersion: "0.3.0"
 description: A Helm chart for Kubernetes that only runs nomad services and uses externally hosted databases.
 name: nomad
-version: 0.1.0
+version: 0.3.0

ops/helm/nomad/templates/api-deployment.yaml

@@ -7,6 +7,7 @@ metadata:
     helm.sh/chart: {{ include "nomad.chart" . }}
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
+    date: {{ .Release.Time.Seconds }}
 spec:
   replicas: {{ .Values.api.replicas }}
   selector:
@@ -73,6 +74,7 @@ spec:
         command: ["python", "-m", "gunicorn.app.wsgiapp", "-b 0.0.0.0:8000", "nomad.api:app"]
       imagePullSecrets:
       - name: {{ .Values.images.secret }}
+      imagePullPolicy: always
       volumes:
       - name: files-volume
         hostPath:

ops/helm/nomad/templates/gui-deployment.yml

@@ -7,6 +7,7 @@ metadata:
     helm.sh/chart: {{ include "nomad.chart" . }}
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
+    date: {{ .Release.Time.Seconds }}
 data:
   nginx.conf: |
     server {
@@ -23,6 +24,7 @@ data:
       "apiBase": "{{ .Values.proxy.external.path }}/api",
       "appBase": "{{ .Values.proxy.external.path }}",
       "appStaticBase": "{{ .Values.proxy.external.path }}",
+      "kibanaBase": "{{ .Values.proxy.external.kibanaPath }}",
       "appDebug": false
     };
 ---
@@ -64,6 +66,7 @@ spec:
           name: log
       imagePullSecrets:
       - name: {{ .Values.images.secret }}
+      imagePullPolicy: always
       volumes:
       - name: nginx-conf
         configMap:

ops/helm/nomad/templates/worker-deployment.yaml

@@ -7,6 +7,7 @@ metadata:
     helm.sh/chart: {{ include "nomad.chart" . }}
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
+    date: {{ .Release.Time.Seconds }}
 spec:
   replicas: {{ .Values.worker.replicas }}
   selector:
@@ -61,6 +62,7 @@ spec:
         command: ["python", "-m", "celery", "worker", "-l", "info", "-A", "nomad.processing"]
       imagePullSecrets:
       - name: {{ .Values.images.secret }}
+      imagePullPolicy: always
       volumes:
       - name: files-volume
         hostPath:

ops/helm/nomad/values.yaml

@@ -53,13 +53,13 @@ gui:
 #  It is run via NodePort service
 proxy:
   port: 80
-  nodePort: 30001
+  nodePort: 30003
   nodeIP: 130.183.207.116
   external:
-    host: "localhost"
-    port: 30001
-    ## Add a path segment with the release name to path, e.g. /nomad/liquid-thunder/...
-    path: "/nomad/kubetest"
+    host: "enc-staging-nomad.esc.rzg.mpg.de"
+    port: 80
+    path: "/fairdi/nomad"
+    kibanaPath: "/fairdi/kibana"
 
 ## configuration of the chart dependency for rabbitmq
 rabbitmq:
@@ -70,7 +70,7 @@ rabbitmq:
     erlangCookie: SWQOKODSQALRPCLNMEQG
 
 ## A common name/prefix for all dbs and indices.
-dbname: nomad_kubetest
+dbname: fairdi_nomad
 
 ## Databases that are not run within the cluster.
 #  To run databases in the cluster, use the nomad-full helm chart.
@@ -96,4 +96,4 @@ kibana:
 
 ## Everything concerning the data that is used by the service
 volumes:
-  files: /scratch/nomad-fair/fs/kubetest
+  files: /scratch/nomad-fair/fs