Commit cc88ac9b authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added a smaller chart for bare bones nomad without dbs.

parent 28ae0cba
......@@ -34,10 +34,10 @@ FSConfig = namedtuple('FSConfig', ['tmp', 'objects'])
RepositoryDBConfig = namedtuple('RepositoryDBConfig', ['host', 'port', 'dbname', 'user', 'password'])
""" Used to configure access to NOMAD-coe repository db. """
ElasticConfig = namedtuple('ElasticConfig', ['host', 'calc_index'])
ElasticConfig = namedtuple('ElasticConfig', ['host', 'port', 'index_name'])
""" Used to configure elastic search. """
MongoConfig = namedtuple('MongoConfig', ['host', 'port', 'users_db'])
MongoConfig = namedtuple('MongoConfig', ['host', 'port', 'db_name'])
""" Used to configure mongo db. """
LogstashConfig = namedtuple('LogstashConfig', ['enabled', 'host', 'tcp_port', 'level'])
......@@ -81,7 +81,8 @@ fs = FSConfig(
)
elastic = ElasticConfig(
host=os.environ.get('NOMAD_ELASTIC_HOST', 'localhost'),
calc_index='calcs'
port=int(os.environ.get('NOMAD_ELASTIC_PORT', 9200)),
index_name=os.environ.get('NOMAD_ELASTIC_INDEX_NAME', 'nomad-fair')
)
repository_db = RepositoryDBConfig(
host=os.environ.get('NOMAD_COE_REPO_DB_HOST', 'localhost'),
......@@ -93,7 +94,7 @@ repository_db = RepositoryDBConfig(
mongo = MongoConfig(
host=os.environ.get('NOMAD_MONGO_HOST', 'localhost'),
port=int(os.environ.get('NOMAD_MONGO_PORT', 27017)),
users_db='users'
db_name=os.environ.get('NOMAD_MONGO_DN_NAME', 'nomad-fair')
)
logstash = LogstashConfig(
enabled=True,
......
......@@ -71,14 +71,15 @@ def setup_logging():
def setup_mongo():
""" Creates connection to mongodb. """
global mongo_client
mongo_client = connect(db=config.mongo.users_db, host=config.mongo.host, port=config.mongo.port)
mongo_client = connect(db=config.mongo.db_name, host=config.mongo.host, port=config.mongo.port)
logger.info('setup mongo connection')
def setup_elastic():
""" Creates connection to elastic search. """
global elastic_client
elastic_client = connections.create_connection(hosts=[config.elastic.host])
elastic_client = connections.create_connection(
hosts=[config.elastic.host], port=[config.elastic.port])
logger.info('setup elastic connection')
try:
......
......@@ -51,7 +51,7 @@ class RepoCalc(ElasticDocument):
instance and only via the :func:`create_from_backend` factory method.
"""
class Index:
name = config.elastic.calc_index
name = config.elastic.index_name
calc_hash = Keyword()
mainfile = Keyword()
......
charts/
\ No newline at end of file
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
apiVersion: v1
appVersion: "1.0"
description: A Helm chart for Kubernetes that runs the whole nomad including databases.
name: nomad-full
version: 0.1.0
dependencies:
- name: rabbitmq
repository: https://kubernetes-charts.storage.googleapis.com/
version: 4.0.1
- name: mongodb
repository: https://kubernetes-charts.storage.googleapis.com/
version: 4.9.1
- name: elasticsearch
repository: https://kubernetes-charts.storage.googleapis.com/
version: 1.15.0
- name: postgresql
repository: https://kubernetes-charts.storage.googleapis.com/
version: 3.1.1
digest: sha256:c6c65e79414429b8b2ecdba6fa2f13628614a0be3c5fff8947c79270a00386fb
generated: 2018-12-12T14:02:40.726497+01:00
dependencies:
- name: rabbitmq
version: "4.0.1"
repository: "https://kubernetes-charts.storage.googleapis.com/"
- name: mongodb
version: "4.9.1"
repository: "https://kubernetes-charts.storage.googleapis.com/"
- name: elasticsearch
version: "1.15.0"
repository: "https://kubernetes-charts.storage.googleapis.com/"
- name: postgresql
version: "3.1.1"
repository: "https://kubernetes-charts.storage.googleapis.com/"
\ No newline at end of file
{{/* vim: set filetype=mustache: */}}
{{/*
Expand the name of the chart.
*/}}
{{- define "nomad.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "nomad.fullname" -}}
{{- if .Values.fullnameOverride -}}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- $name := default .Chart.Name .Values.nameOverride -}}
{{- if contains $name .Release.Name -}}
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "nomad.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
{{- end -}}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "nomad.fullname" . }}-api
labels:
app.kubernetes.io/name: {{ include "nomad.name" . }}-api
helm.sh/chart: {{ include "nomad.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
spec:
replicas: {{ .Values.api.replicas }}
selector:
matchLabels:
app.kubernetes.io/name: {{ include "nomad.name" . }}-api
app.kubernetes.io/instance: {{ .Release.Name }}
template:
metadata:
labels:
app.kubernetes.io/name: {{ include "nomad.name" . }}-api
app.kubernetes.io/instance: {{ .Release.Name }}
spec:
containers:
- name: {{ include "nomad.name" . }}-api
image: "{{ .Values.images.nomad.name }}:{{ .Values.images.nomad.tag }}"
volumeMounts:
- mountPath: /app/.volumes/fs
name: files-volume
env:
- name: NOMAD_SERVICE
value: "api"
- name: NOMAD_LOGSTASH_HOST
value: {{ .Values.logstash.host }}
- name: NOMAD_LOGSTASH_TCPPORT
value: {{ .Values.logstash.port }}
- name: NOMAD_CONSOLE_LOGLEVEL
value: {{ .Values.api.console_loglevel }}
- name: NOMAD_LOGSTASH_LEVEL
value: {{ .Values.api.logstash_loglevel }}
- name: NOMAD_API_PORT
value: {{ .Values.api.port }}
- name: NOMAD_API_SECRET
value: {{ .Values.api.secret }}
imagePullSecrets:
- name: {{ .Values.images.secret }}
volumes:
- name: files-volume
hostPath:
path: {{ .Values.volumes.files }}
type: Directory
apiVersion: v1
kind: Service
metadata:
name: {{ include "nomad.fullname" . }}-api
labels:
app.kubernetes.io/name: {{ include "nomad.name" . }}-api
helm.sh/chart: {{ include "nomad.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
spec:
type: {{ .Values.api.service_type }}
ports:
- port: {{ .Values.api.port }}
targetPort: http
protocol: TCP
name: http
selector:
app.kubernetes.io/name: {{ include "nomad.name" . }}-api
app.kubernetes.io/instance: {{ .Release.Name }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "nomad.fullname" . }}-worker
labels:
app.kubernetes.io/name: {{ include "nomad.name" . }}-worker
helm.sh/chart: {{ include "nomad.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
spec:
replicas: {{ .Values.worker.replicas }}
selector:
matchLabels:
app.kubernetes.io/name: {{ include "nomad.name" . }}-worker
app.kubernetes.io/instance: {{ .Release.Name }}
template:
metadata:
labels:
app.kubernetes.io/name: {{ include "nomad.name" . }}-worker
app.kubernetes.io/instance: {{ .Release.Name }}
spec:
containers:
- name: {{ include "nomad.name" . }}-worker
image: "{{ .Values.images.nomad.name }}:{{ .Values.images.nomad.tag }}"
volumeMounts:
- mountPath: /app/.volumes/fs
name: files-volume
env:
- name: NOMAD_SERVICE
value: "worker"
- name: NOMAD_LOGSTASH_HOST
value: {{ .Values.logstash.host }}
- name: NOMAD_LOGSTASH_TCPPORT
value: {{ .Values.logstash.port }}
- name: NOMAD_CONSOLE_LOGLEVEL
value: {{ .Values.worker.console_loglevel }}
- name: NOMAD_LOGSTASH_LEVEL
value: {{ .Values.worker.logstash_loglevel }}
imagePullSecrets:
- name: {{ .Values.images.secret }}
volumes:
- name: files-volume
hostPath:
path: {{ .Values.volumes.files }}
type: Directory
## Default values for nomad@FAIRDI
## Everything concerning the container images to be used
images:
## The kubernetes docker-registry secret that can be used to access the registry
# with the container image in it.
# It can be created via:
# kubectl create secret docker-registry gitlab-mpcdf --docker-server=gitlab-registry.mpcdf.mpg.de --docker-username=<your-user-name > --docker-password=<yourpass> --docker-email=<email>
secret: gitlab-mpcdf
## The nomad image with all nomad relavant python code. Used by api and worker service.
nomad:
## The docker container image name without tag
name: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair
## The docker container image tag
tag: latest
pullPolicy: IfNotPresent
## Everthing concerning the nomad api
api:
replicas: 1
port: 8000
console_loglevel: INFO
logstash_loglevel: INFO
service_type: ClusterIP
## Secret used as cryptographic seed
secret: "defaultApiSecret"
## Everthing concerning the nomad worker
worker:
replicas: 2
console_loglevel: INFO
logstash_loglevel: INFO
rabbitmq:
image.pullSecrets: nil
rbacEnabled: false
rabbitmq:
username: rabbitmq
password: rabbitmq
erlangCookie: SWQOKODSQALRPCLNMEQG
persistence.enabled: false
securityContext.enabled: false
mongodb:
image.pullSecrets: nil
usePassword: false
persistence.enabled: true
securityContext.enabled: false
elasticsearch:
client.replicas: 2
master.replicas: 1
data.replicas: 1
cluster.env: {MINIMUM_MASTER_NODES=""1"}
postgresql:
postgresqlUsername: "postgres"
postgresqlPassword: "nomad"
postgresqlDatabase: "nomad"
securityContext.enabled: false
logstash:
port: 5000
host: "enc-preprocessing-nomad.esc"
## Everthing concerning the data that is used by the service
volumes:
files: /nomad/nomadlab/nomad-FAIRDI/files
apiVersion: v1
appVersion: "1.0"
description: A Helm chart for Kubernetes
description: A Helm chart for Kubernetes that only runs nomad services and uses externally hosted databases.
name: nomad
version: 0.1.0
......@@ -2,14 +2,5 @@ dependencies:
- name: rabbitmq
repository: https://kubernetes-charts.storage.googleapis.com/
version: 4.0.1
- name: mongodb
repository: https://kubernetes-charts.storage.googleapis.com/
version: 4.9.1
- name: elasticsearch
repository: https://kubernetes-charts.storage.googleapis.com/
version: 1.15.0
- name: postgresql
repository: https://kubernetes-charts.storage.googleapis.com/
version: 3.1.1
digest: sha256:c6c65e79414429b8b2ecdba6fa2f13628614a0be3c5fff8947c79270a00386fb
generated: 2018-12-12T14:02:40.726497+01:00
digest: sha256:91440e2bb5d4645a88c222836626306ef3d4e0c1e18a46c1442a2b3081655631
generated: 2018-12-13T14:16:27.606343+01:00
dependencies:
- name: rabbitmq
version: "4.0.1"
repository: "https://kubernetes-charts.storage.googleapis.com/"
- name: mongodb
version: "4.9.1"
repository: "https://kubernetes-charts.storage.googleapis.com/"
- name: elasticsearch
version: "1.15.0"
repository: "https://kubernetes-charts.storage.googleapis.com/"
- name: postgresql
version: "3.1.1"
repository: "https://kubernetes-charts.storage.googleapis.com/"
\ No newline at end of file
......@@ -40,6 +40,26 @@ spec:
value: "{{ .Values.api.port }}"
- name: NOMAD_API_SECRET
value: "{{ .Values.api.secret }}"
- name: NOMAD_RABBITMQ_HOST
value: "{{ .Release.Name }}-rabbitmq"
- name: NOMAD_ELASTIC_HOST
value: "{{ .Values.elastic.host }}"
- name: NOMAD_ELASTIC_PORT
value: "{{ .Values.elastic.port }}"
- name: NOMAD_ELASTIC_INDEX_NAME
value: "{{ .Release.Name }}-nomad"
- name: NOMAD_MONGO_HOST
value: "{{ .Values.mongo.host }}"
- name: NOMAD_MONGO_PORT
value: "{{ .Values.mongo.port }}"
- name: NOMAD_MONGO_DB_NAME
value: "{{ .Release.Name }}-nomad"
- name: NOMAD_COE_REPO_DB_HOST
value: "{{ .Values.postgres.host }}"
- name: NOMAD_COE_REPO_DB_PORT
value: "{{ .Values.postgres.port }}"
- name: NOMAD_COE_REPO_DB_NAME
value: "{{ .Release.Name }}-nomad"
command: ["python", "-m", "gunicorn.app.wsgiapp", "-b 0.0.0.0:8000", "nomad.api:app"]
imagePullSecrets:
- name: {{ .Values.images.secret }}
......
......@@ -36,6 +36,26 @@ spec:
value: "{{ .Values.worker.console_loglevel }}"
- name: NOMAD_LOGSTASH_LEVEL
value: "{{ .Values.worker.logstash_loglevel }}"
- name: NOMAD_RABBITMQ_HOST
value: "{{ .Release.Name }}-rabbitmq"
- name: NOMAD_ELASTIC_HOST
value: "{{ .Values.elastic.host }}"
- name: NOMAD_ELASTIC_PORT
value: "{{ .Values.elastic.port }}"
- name: NOMAD_ELASTIC_INDEX_NAME
value: "{{ .Release.Name }}-nomad"
- name: NOMAD_MONGO_HOST
value: "{{ .Values.mongo.host }}"
- name: NOMAD_MONGO_PORT
value: "{{ .Values.mongo.port }}"
- name: NOMAD_MONGO_DB_NAME
value: "{{ .Release.Name }}-nomad"
- name: NOMAD_COE_REPO_DB_HOST
value: "{{ .Values.postgres.host }}"
- name: NOMAD_COE_REPO_DB_PORT
value: "{{ .Values.postgres.port }}"
- name: NOMAD_COE_REPO_DB_NAME
value: "{{ .Release.Name }}-nomad"
command: ["python", "-m", "celery", "worker", "-l", "info", "-A", "nomad.processing"]
imagePullSecrets:
- name: {{ .Values.images.secret }}
......
......@@ -32,6 +32,7 @@ worker:
console_loglevel: INFO
logstash_loglevel: INFO
## configuration of the chart dependency for rabbitmq
rabbitmq:
image.pullSecrets: nil
rbacEnabled: false
......@@ -42,37 +43,25 @@ rabbitmq:
plugins: |-
[].
configuration: |-
persistence:
enabled: false
mongodb:
usePassword: false
securityContext:
enabled: false
mongo:
host: enc-preprocessing-nomad.esc
port: 27017
elasticsearch:
client:
replicas: 2
master:
replicas: 1
data:
replicas: 1
cluster:
env:
MINIMUM_MASTER_NODES: 1
elastic:
host: enc-preprocessing-nomad.esc
port: 9200
postgresql:
postgresqlUsername: "postgresd"
postgresqlPassword: "nomad"
postgresqlDatabase: "nomad"
securityContext:
enabled: false
postgres:
host: enc-preprocessing-nomad.esc
port: 5432
logstash:
port: "5000"
host: "enc-preprocessing-nomad.esc"
port: 5000
host: enc-preprocessing-nomad.esc
## Everthing concerning the data that is used by the service
## Everything concerning the data that is used by the service
volumes:
files: /nomad/nomadlab/nomad-FAIRDI/files
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment