.gitlab-ci.yml

# default installed image for docker executor is: python:3.6
# using an image that can do git, docker, docker-compose
image: youpy/docker-compose-git

# Uncomment the next lines, to run each pipline/job in its own docker environment.
# Otherwise, it will use the docker of the gitlab runner host (e.g. enc-preprocessing...).
# This will give it access to a persitent layer cache, which will not be available
# with the docker service.
# services:
#   - docker:dind

stages:
  - build
  - test
  - integration
  - release
  - deploy

variables:
  TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:${CI_COMMIT_REF_NAME}
  RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest
  FRONTEND_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:${CI_COMMIT_REF_NAME}
  FRONTEND_RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:latest
  RAWAPI_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/rawapi:${CI_COMMIT_REF_NAME}
  RAWAPI_RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/rawapi:latest

  KUBECONFIG: /etc/deploy/config
  STAGING_NAMESPACE: nomad-fairdi

build:
  stage: build
  before_script:
    - git submodule sync
    - git submodule update --init
  script:
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
    - docker build --no-cache -t $TEST_IMAGE .
    - docker push $TEST_IMAGE
  except:
    - /^dev-.*$/


buildgui:
  stage: build
  script:
    - cd gui
    - ./gitinfo.sh
    - ls -la src/gitinfo.json
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
    - docker build -t $FRONTEND_TEST_IMAGE .
    - docker push $FRONTEND_TEST_IMAGE
  except:
    - /^dev-.*$/


buildrawapi:
  stage: build
  script:
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
    - docker build -t $RAWAPI_TEST_IMAGE -f rawapi.Dockerfile .
    - docker push $RAWAPI_TEST_IMAGE
  only:
    - rawapi


linting:
  stage: test
  image: $TEST_IMAGE
  script:
    - cd /app
    - python -m pycodestyle --ignore=E501,E701 nomad tests
    - python -m pylint --load-plugins=pylint_mongoengine nomad tests
    - python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests
  except:
    - /^dev-.*$/

tests:
  stage: test
  image: $TEST_IMAGE
  services:
    - postgres:latest
      # this will cause a warning, as the gitlab ci runner health check will test the wrong
      # port on rabbitmq container:
      # https://gitlab.com/gitlab-org/gitlab-runner/issues/3163
    - rabbitmq
    - name: docker.elastic.co/elasticsearch/elasticsearch:6.3.2
      alias: elastic
      # fix issue with running elastic in gitlab ci runner:
      # https://gitlab.com/gitlab-org/gitlab-ce/issues/42214
      command: [ "bin/elasticsearch", "-Ediscovery.type=single-node" ]
  variables:
    POSTGRES_DB: nomad
    POSTGRES_USER: postgres
    POSTGRES_PASSWORD: nomad
    RABBITMQ_ERLANG_COOKIE: SWQOKODSQALRPCLNMEQG
    RABBITMQ_DEFAULT_USER: rabbitmq
    RABBITMQ_DEFAULT_PASS: rabbitmq
    RABBITMQ_DEFAULT_VHOST: /
    NOMAD_COE_REPO_DB_HOST: postgres
    NOMAD_RABBITMQ_HOST: rabbitmq
    NOMAD_ELASTIC_HOST: elastic
    NOMAD_MONGO_HOST: mongo
  script:
    - cd /app
    - python -m pytest --cov=nomad -sv tests
  except:
    - /^dev-.*$/

# does currently not work, current GitLab CI runner does not network services with each other
# integration-tests:
#   stage: integration
#   services:
#     - postgres:latest
#     - mongo:latest
#       # this will cause a warning, as the gitlab ci runner health check will test the wrong
#       # port on rabbitmq container:
#       # https://gitlab.com/gitlab-org/gitlab-runner/issues/3163
#     - rabbitmq
#     - name: docker.elastic.co/elasticsearch/elasticsearch:6.3.2
#       alias: elastic
#       # fix issue with running elastic in gitlab ci runner:
#       # https://gitlab.com/gitlab-org/gitlab-ce/issues/42214
#       command: [ "bin/elasticsearch", "-Ediscovery.type=single-node" ]
#     - name: $TEST_IMAGE
#       alias: api
#       command: ["python", "-m", "nomad.client", "run", "api"]
#     - name: $TEST_IMAGE
#       alias: worker
#       command: ["python", "-m", "nomad.client", "run", "worker"]
#   variables:
#     POSTGRES_DB: nomad
#     POSTGRES_USER: postgres
#     POSTGRES_PASSWORD: nomad
#     RABBITMQ_ERLANG_COOKIE: SWQOKODSQALRPCLNMEQG
#     RABBITMQ_DEFAULT_USER: rabbitmq
#     RABBITMQ_DEFAULT_PASS: rabbitmq
#     RABBITMQ_DEFAULT_VHOST: /
#     NOMAD_COE_REPO_DB_HOST: postgres
#     NOMAD_RABBITMQ_HOST: rabbitmq
#     NOMAD_ELASTIC_HOST: elastic
#     NOMAD_MONGO_HOST: mongo
#   script:
#     - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
#     - integration/test_integration.sh
#   except:
#     - /^dev-.*$/

release:
  stage: release
  script:
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
    - docker pull $TEST_IMAGE
    - docker tag $TEST_IMAGE $RELEASE_IMAGE
    - docker push $RELEASE_IMAGE
    - docker pull $FRONTEND_TEST_IMAGE
    - docker tag $FRONTEND_TEST_IMAGE $FRONTEND_RELEASE_IMAGE
    - docker push $FRONTEND_RELEASE_IMAGE
  except:
    - /^dev-.*$/
  when: manual

release_rawapi:
  stage: release
  script:
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
    - docker pull $RAWAPI_TEST_IMAGE
    - docker tag $RAWAPI_TEST_IMAGE $RAWAPI_RELEASE_IMAGE
    - docker push $RAWAPI_RELEASE_IMAGE
  only:
    - rawapi
  when: manual

deploy_rawapi:
  stage: deploy
  script:
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
    - cd ops/docker-compose/rawapi
    # secret .env file is stored on the "production" machine and is bind mounted into the
    # runner container at /nomad/config/.env
    - cp /nomad/config/.rawapi_env .env
    - docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d
  only:
    - rawapi
  when: manual

deploy:
  stage: deploy
  image: dtzar/helm-kubectl
  before_script:
    - mkdir -p /etc/deploy
    # kube_config is a CI/CD variable set in GitLab GUI
    - echo $CI_KUBE_CONFIG | base64 -d > /etc/deploy/config
    - helm init --client-only
    - helm repo add stable https://kubernetes-charts.storage.googleapis.com/
    - helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com/
    - helm repo update
  script:
    - cd ops/helm/nomad
    - export KUBECONFIG=/etc/deploy/config
    - helm dep build
    - export NOMAD_VERSION="$(grep "version" Chart.yaml | cut -d" " -f2)"
    - export NUMERIC_VERSION="$(echo ${NOMAD_VERSION} | cut -d"." -f1-2)"
    - export RELEASE_NAME="nomad-v${NOMAD_VERSION//./-}"
    - export DEPLOYS=$(helm ls | grep $RELEASE_NAME | wc -l)
    - export EXTERNAL_PATH="/fairdi/nomad/v${NOMAD_VERSION}"
    - export DBNAME="fairdi_nomad_v${NOMAD_VERSION//./_}"
    - export FILES_PATH="/scratch/nomad-fair/fs/nomad_v${NOMAD_VERSION}"
    - if [ ${DEPLOYS}  -eq 0 ]; then
        helm install --name=${RELEASE_NAME} . --namespace=${STAGING_NAMESPACE}
          --set api.disableReset="false"
          --set proxy.nodePort="300${NUMERIC_VERSION//./}"
          --set proxy.external.path=${EXTERNAL_PATH}
          --set dbname=${DBNAME}
          --set worker.replicas=1
          --set worker.memrequest=32
          --set volumes.files=${FILES_PATH};
      else
        helm upgrade ${RELEASE_NAME} . --namespace=${STAGING_NAMESPACE} --recreate-pods;
      fi
  except:
    - /^dev-.*$/
  when: manual