Commit 799c70f9 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Removed old rawapi artifacts.

parent 7e8f69af
......@@ -23,11 +23,7 @@ variables:
FRONTEND_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:test_${CI_COMMIT_REF_NAME}
FRONTEND_RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:${CI_COMMIT_REF_NAME}
FRONTEND_LATEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:latest
RAWAPI_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/rawapi:${CI_COMMIT_REF_NAME}
RAWAPI_RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/rawapi:latest
KUBECONFIG: /etc/deploy/config
STAGING_NAMESPACE: nomad-fairdi
build:
stage: build
......@@ -59,32 +55,6 @@ buildgui:
- /^dev-.*$/
- tags
buildrawapi:
stage: build
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker build -t $RAWAPI_TEST_IMAGE -f rawapi.Dockerfile .
- docker push $RAWAPI_TEST_IMAGE
only:
- rawapi
linting:
stage: test
image: $TEST_IMAGE
script:
- cd /app
- python -m pycodestyle --ignore=E501,E701 nomad tests
- python -m pylint --load-plugins=pylint_mongoengine nomad tests
- python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests
except:
refs:
- tags
variables:
- $CI_COMMIT_REF_NAME =~ /^dev-.*$/
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
tests:
stage: test
image: $TEST_IMAGE
......@@ -121,45 +91,6 @@ tests:
- $CI_COMMIT_REF_NAME =~ /^dev-.*$/
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
# does currently not work, current GitLab CI runner does not network services with each other
# integration-tests:
# stage: integration
# services:
# - postgres:latest
# - mongo:latest
# # this will cause a warning, as the gitlab ci runner health check will test the wrong
# # port on rabbitmq container:
# # https://gitlab.com/gitlab-org/gitlab-runner/issues/3163
# - rabbitmq
# - name: docker.elastic.co/elasticsearch/elasticsearch:6.3.2
# alias: elastic
# # fix issue with running elastic in gitlab ci runner:
# # https://gitlab.com/gitlab-org/gitlab-ce/issues/42214
# command: [ "bin/elasticsearch", "-Ediscovery.type=single-node" ]
# - name: $TEST_IMAGE
# alias: api
# command: ["python", "-m", "nomad.client", "run", "api"]
# - name: $TEST_IMAGE
# alias: worker
# command: ["python", "-m", "nomad.client", "run", "worker"]
# variables:
# POSTGRES_DB: nomad
# POSTGRES_USER: postgres
# POSTGRES_PASSWORD: nomad
# RABBITMQ_ERLANG_COOKIE: SWQOKODSQALRPCLNMEQG
# RABBITMQ_DEFAULT_USER: rabbitmq
# RABBITMQ_DEFAULT_PASS: rabbitmq
# RABBITMQ_DEFAULT_VHOST: /
# NOMAD_REPOSITORY_DB_HOST: postgres
# NOMAD_RABBITMQ_HOST: rabbitmq
# NOMAD_ELASTIC_HOST: elastic
# NOMAD_MONGO_HOST: mongo
# script:
# - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
# - integration/test_integration.sh
# except:
# - /^dev-.*$/
release:
stage: release
script:
......@@ -187,31 +118,7 @@ release_version:
only:
- tags
release_rawapi:
stage: release
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker pull $RAWAPI_TEST_IMAGE
- docker tag $RAWAPI_TEST_IMAGE $RAWAPI_RELEASE_IMAGE
- docker push $RAWAPI_RELEASE_IMAGE
only:
- rawapi
when: manual
deploy_rawapi:
stage: deploy
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- cd ops/docker-compose/rawapi
# secret .env file is stored on the "production" machine and is bind mounted into the
# runner container at /nomad/config/.env
- cp /nomad/config/.rawapi_env .env
- docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d
only:
- rawapi
when: manual
deploy:
deploy_coe_staging:
stage: deploy
image: dtzar/helm-kubectl
before_script:
......@@ -226,31 +133,7 @@ deploy:
- cd ops/helm/nomad
- export KUBECONFIG=/etc/deploy/config
- helm dep build
- export NOMAD_VERSION="$(grep "version" Chart.yaml | cut -d" " -f2)"
- export NUMERIC_VERSION="$(echo ${NOMAD_VERSION} | cut -d"." -f1-2)"
- export RELEASE_NAME="nomad-v${NOMAD_VERSION//./-}"
- export DEPLOYS=$(helm ls | grep $RELEASE_NAME | wc -l)
- export EXTERNAL_PATH="/fairdi/nomad/v${NOMAD_VERSION}"
- export DBNAME="fairdi_nomad_v${NOMAD_VERSION//./_}"
- export FILES_PATH="/nomad/fairdi/nomad_v${NOMAD_VERSION}/fs"
- if [ ${DEPLOYS} -eq 0 ]; then
helm install --name=${RELEASE_NAME} . --namespace=${STAGING_NAMESPACE}
--set images.nomad.tag=${CI_COMMIT_REF_NAME}
--set images.frontend.tag=${CI_COMMIT_REF_NAME}
--set api.disableReset="false"
--set proxy.nodePort="300${NUMERIC_VERSION//./}"
--set proxy.external.path=${EXTERNAL_PATH}
--set dbname=${DBNAME}
--set worker.replicas=1
--set worker.memrequest=32
--set volumes.public=${FILES_PATH}/public
--set volumes.staging=${FILES_PATH}/staging
--set volumes.tmp=${FILES_PATH}/tmp;
else
helm upgrade ${RELEASE_NAME} . --namespace=${STAGING_NAMESPACE}
--set images.nomad.tag=${CI_COMMIT_REF_NAME}
--set images.frontend.tag=${CI_COMMIT_REF_NAME}
--recreate-pods;
fi
only:
- tags
- helm upgrade nomad-coe-staging . --recreate-pods;
except:
- /^dev-.*$/
when: manual
Contains all files to run the rawapi with docker-compose on a single docker, both
for development and production.
We use docker-compose overrides to modify config for development and production. Example:
```
docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d
```
The different overrides are:
- *.prod.yml, production (currently on enc-preprocessing-nomad.esc)
You have to create a .env file with the variable `RAW_FILE_DIR` to determine the
directory with the rawfiles: `<RAW_FILE_DIR>/data/R*.zip` .
\ No newline at end of file
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
version: '3.4'
services:
# nomad rawapi
api:
restart: always
image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/rawapi:latest
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
version: '3.4'
services:
# nomad rawapi
api:
image: nomad/rawapi
container_name: nomad_rawapi
environment:
NOMAD_API_BASE_PATH: /nomad/rawapi
NOMAD_SERVICE: rawapi
volumes:
- ${RAW_FILE_DIR}:/raw:ro
ports:
- 18001:8000
command: python -m gunicorn.app.wsgiapp -b 0.0.0.0:8000 nomad.api:app
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
apiVersion: v1
appVersion: "1.0"
description: A Helm chart for Kubernetes
name: rawapi
version: 0.1.0
{{/* vim: set filetype=mustache: */}}
{{/*
Expand the name of the chart.
*/}}
{{- define "rawapi.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "rawapi.fullname" -}}
{{- if .Values.fullnameOverride -}}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- $name := default .Chart.Name .Values.nameOverride -}}
{{- if contains $name .Release.Name -}}
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "rawapi.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
{{- end -}}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "rawapi.fullname" . }}
labels:
app.kubernetes.io/name: {{ include "rawapi.name" . }}
helm.sh/chart: {{ include "rawapi.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: {{ include "rawapi.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
template:
metadata:
labels:
app.kubernetes.io/name: {{ include "rawapi.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
spec:
containers:
- name: nomad-rawapi
image: "{{ .Values.image.name }}:{{ .Values.image.tag }}"
volumeMounts:
- mountPath: /raw
name: raw-volume
imagePullSecrets:
- name: {{ .Values.image.secret }}
volumes:
- name: raw-volume
hostPath:
path: {{ .Values.volumes.raw }}
type: Directory
\ No newline at end of file
apiVersion: v1
kind: Service
metadata:
name: {{ include "rawapi.fullname" . }}
labels:
app.kubernetes.io/name: {{ include "rawapi.name" . }}
helm.sh/chart: {{ include "rawapi.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
spec:
selector:
app.kubernetes.io/name: {{ include "rawapi.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
ports:
- protocol: TCP
port: 8000
targetPort: 8000
nodePort: {{ .Values.service.nodePort }}
type: NodePort
## Default values for the nomad@FAIRDI rawapi
## Everything concerning the container image to be used
image:
## The kubernetes docker-registry secret that can be used to access the registry
# with the container image in it.
# It can be created via:
# kubectl create secret docker-registry gitlab-mpcdf --docker-server=gitlab-registry.mpcdf.mpg.de --docker-username=<your-user-name > --docker-password=<yourpass> --docker-email=<email>
secret: gitlab-mpcdf
## The docker container image name without tag
name: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/rawapi
## The docker container image tag
tag: latest
pullPolicy: IfNotPresent
## Everthing concerning the service operation
service:
## Currenty rawapi is configured to expose the service via a kubernetes NodePort
nodePort: 30001
## Everthing concerning the data that is used by the service
volumes:
## The hostPath that contains the raw file archives. This directory must have a
# subdirectory called 'data' with the actual files in it. The files must be situated
# in 3-digit prefix sub directories; example: data/H3j/H3ju22kjHkls18l.zip
raw: /nomad/nomadlab/raw-data
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This dockerfile creates a limited api container that only runs the raw file api endpoint
# We use slim for the final image
FROM python:3.6-slim as final
# First, build everything in a build image
FROM python:3.6-stretch as build
RUN mkdir /install
WORKDIR /install
# We also install the -dev dependencies, to use this image for test and qa
RUN pip install --upgrade pip
COPY rawapi.requirements.txt rawapi.requirements.txt
RUN pip install -r rawapi.requirements.txt
# do that after the dependencies to use docker's layer caching
COPY . /install
RUN echo "from .app import app\nfrom . import raw" > /install/nomad/api/__init__.py
RUN pip install .
RUN \
find /usr/local/lib/python3.6/ -name 'tests' ! -path '*/networkx/*' -exec rm -r '{}' + && \
find /usr/local/lib/python3.6/ -name 'test' -exec rm -r '{}' + && \
find /usr/local/lib/python3.6/site-packages/ -name '*.so' -print -exec sh -c 'file "{}" | grep -q "not stripped" && strip -s "{}"' \;
# Second, create a slim final image
FROM final
# copy the sources for tests, coverage, qa, etc.
COPY --from=build /install /app
WORKDIR /app
# transfer installed packages from dependency stage
COPY --from=build /usr/local/lib/python3.6/site-packages /usr/local/lib/python3.6/site-packages
RUN mkdir -p /raw
RUN useradd -ms /bin/bash nomad
RUN chown -R nomad /app
USER nomad
ENV NOMAD_FILES_OBJECTS_DIR /raw
ENV NOMAD_FILES_RAW_BUCKET data
ENV NOMAD_SERVICE rawapi
CMD python -m gunicorn.app.wsgiapp -b 0.0.0.0:8000 nomad.api:app
VOLUME /raw
EXPOSE 8000
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment