From 02b38910a4e7825d56017d244dd1ae229d0332fb Mon Sep 17 00:00:00 2001
From: Markus Scheidgen <markus.scheidgen@gmail.com>
Date: Wed, 3 Jul 2024 12:19:46 +0200
Subject: [PATCH] Added jupyter based admin console to k8s deployments.
Changelog: Added
---
Dockerfile | 2 +
ops/kubernetes/nomad-prod-util.yaml | 3 +
.../templates/adminconsole/deployment.yaml | 164 ++++++++++++++++++
ops/kubernetes/nomad/values.yaml | 50 ++++++
ops/kubernetes/values.yaml | 19 +-
5 files changed, 236 insertions(+), 2 deletions(-)
create mode 100644 ops/kubernetes/nomad/templates/adminconsole/deployment.yaml
diff --git a/Dockerfile b/Dockerfile
index c6fc706aff..abc89c3aa5 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -217,7 +217,9 @@ COPY --chown=nomad:1000 nomad/jupyterhub_config.py ./nomad/jupyterhub_config.py
COPY --chown=nomad:1000 examples/data/uploads /app/examples/data/uploads
COPY --chown=nomad:1000 --from=builder /usr/local/lib/python3.9/site-packages /usr/local/lib/python3.9/site-packages
COPY --chown=nomad:1000 --from=builder /usr/local/share/jupyterhub /usr/local/share/jupyterhub
+COPY --chown=nomad:1000 --from=builder /usr/local/share/jupyter /usr/local/share/jupyter
COPY --chown=nomad:1000 --from=builder /usr/local/bin/nomad /usr/local/bin/nomad
+COPY --chown=nomad:1000 --from=builder /usr/local/bin/jupyter* /usr/local/bin/
RUN mkdir -p /app/.volumes/fs \
&& chown -R nomad:1000 /app \
diff --git a/ops/kubernetes/nomad-prod-util.yaml b/ops/kubernetes/nomad-prod-util.yaml
index 4966a53dd7..c50943af44 100644
--- a/ops/kubernetes/nomad-prod-util.yaml
+++ b/ops/kubernetes/nomad-prod-util.yaml
@@ -45,3 +45,6 @@ nomad:
memory: "256Gi"
requests:
memory: "8Gi"
+
+ adminconsole:
+ enabled: true
diff --git a/ops/kubernetes/nomad/templates/adminconsole/deployment.yaml b/ops/kubernetes/nomad/templates/adminconsole/deployment.yaml
new file mode 100644
index 0000000000..a8bb7dba72
--- /dev/null
+++ b/ops/kubernetes/nomad/templates/adminconsole/deployment.yaml
@@ -0,0 +1,164 @@
+{{- if .Values.nomad.adminconsole.enabled -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ include "nomad.fullname" . }}-adminconsole
+ labels:
+ {{- include "nomad.labels" . | nindent 4 }}
+ app.kubernetes.io/component: adminconsole
+spec:
+ replicas: {{ .Values.nomad.adminconsole.replicaCount }}
+ selector:
+ matchLabels:
+ {{- include "nomad.selectorLabels" . | nindent 6 }}
+ app.kubernetes.io/component: adminconsole
+ template:
+ metadata:
+ {{- with .Values.nomad.adminconsole.podAnnotations }}
+ annotations:
+ {{- toYaml . | nindent 8 }}
+ {{- if .Values.roll }}
+ rollme: {{ randAlphaNum 5 | quote }}
+ {{- end }}
+ {{- else }}
+ {{- if .Values.roll }}
+ annotations:
+ rollme: {{ randAlphaNum 5 | quote }}
+ {{- end }}
+ {{- end }}
+ labels:
+ {{- include "nomad.labels" . | nindent 8 }}
+ {{- with .Values.nomad.podLabels }}
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ app.kubernetes.io/component: adminconsole
+ spec:
+ {{- with .Values.nomad.imagePullSecrets }}
+ imagePullSecrets:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ serviceAccountName: {{ include "nomad.serviceAccountName" . }}
+ {{- with .Values.nomad.adminconsole.podSecurityContext }}
+ securityContext:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ containers:
+ - name: {{ .Chart.Name }}-adminconsole
+ {{- with .Values.nomad.adminconsole.securityContext }}
+ securityContext:
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ image: "{{ .Values.nomad.image.repository }}:{{ .Values.nomad.image.tag | default .Chart.AppVersion }}"
+ imagePullPolicy: {{ .Values.nomad.image.pullPolicy }}
+ {{- with .Values.nomad.adminconsole.resources }}
+ resources:
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ volumeMounts:
+ - mountPath: /app/nomad.yaml
+ name: nomad-conf
+ subPath: nomad.yaml
+ - mountPath: /app/.volumes/fs/public
+ name: public-volume
+ - mountPath: /app/.volumes/fs/staging
+ name: staging-volume
+ - mountPath: /nomad
+ name: nomad-volume
+ {{- with .Values.nomad.volumeMounts }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ env:
+ - name: JUPYTER_RUNTIME_DIR
+ value: /tmp
+ - name: JUPYTER_DATA_DIR
+ value: /tmp
+ - name: CELERY_ACKS_LATE
+ value: "True"
+ - name: NOMAD_META_SERVICE
+ value: "adminconsole"
+ - name: NOMAD_CONSOLE_LOG_LEVEL
+ value: "{{ .Values.nomad.config.worker.console_loglevel }}"
+ - name: NOMAD_LOGSTASH_LEVEL
+ value: "{{ .Values.nomad.config.worker.logstash_loglevel }}"
+ - name: NOMAD_CELERY_NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: spec.nodeName
+ {{- if .Values.nomad.config.api.apiSecret }}
+ - name: NOMAD_SERVICES_API_SECRET
+ valueFrom:
+ secretKeyRef:
+ name: {{ .Values.nomad.config.api.apiSecret}}
+ key: password
+ {{- end }}
+ {{- if .Values.nomad.config.keycloak.clientSecret }}
+ - name: NOMAD_KEYCLOAK_CLIENT_SECRET
+ valueFrom:
+ secretKeyRef:
+ name: {{ .Values.nomad.config.keycloak.clientSecret }}
+ key: password
+ {{- end }}
+ {{- if .Values.nomad.config.keycloak.passwordSecret }}
+ - name: NOMAD_KEYCLOAK_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: {{ .Values.nomad.config.keycloak.passwordSecret }}
+ key: password
+ {{- end }}
+ command: ["jupyter", "lab", "--notebook-dir", "/nomad/adminconsole"]
+ livenessProbe:
+ exec:
+ command:
+ - bash
+ - -c
+ - NOMAD_LOGSTASH_LEVEL=WARNING python -m celery -A nomad.processing status | grep "${NOMAD_CELERY_NODE_NAME}:.*OK"
+ initialDelaySeconds: 30
+ periodSeconds: 120
+ timeoutSeconds: 60
+ readinessProbe:
+ exec:
+ command:
+ - bash
+ - -c
+ - NOMAD_LOGSTASH_LEVEL=WARNING python -m celery -A nomad.processing status | grep "${NOMAD_CELERY_NODE_NAME}:.*OK"
+ initialDelaySeconds: 30
+ periodSeconds: 120
+ timeoutSeconds: 60
+ volumes:
+ {{- with .Values.nomad.volumes }}
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ - name: nomad-conf
+ configMap:
+ name: {{ include "nomad.fullname" . }}-configmap
+ - name: public-volume
+ hostPath:
+ path: {{ .Values.nomad.config.volumes.public }}
+ # type: Directory
+ - name: staging-volume
+ {{ if (eq .Values.nomad.config.worker.storage "memory") }}
+ emptyDir:
+ medium: 'Memory'
+ {{ else }}
+ hostPath:
+ path: {{ .Values.nomad.config.volumes.staging}}
+ # type: Directory
+ {{ end }}
+ - name: nomad-volume
+ hostPath:
+ path: {{ .Values.nomad.config.volumes.nomad }}
+ # type: Directory
+
+ {{- with .Values.nomad.adminconsole.nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.nomad.adminconsole.affinity }}
+ affinity:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.nomad.adminconsole.tolerations }}
+ tolerations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+{{- end }}
\ No newline at end of file
diff --git a/ops/kubernetes/nomad/values.yaml b/ops/kubernetes/nomad/values.yaml
index 33955dbdfc..b5bbfb6dc9 100644
--- a/ops/kubernetes/nomad/values.yaml
+++ b/ops/kubernetes/nomad/values.yaml
@@ -353,6 +353,56 @@ nomad:
podAnnotations: {}
podLabels: {}
+ ## Everything concerning the nomad adminconsole
+ adminconsole:
+ enabled: false
+
+ replicaCount: 1
+
+ podSecurityContext: {}
+ # fsGroup: 2000
+
+ securityContext: {}
+ # capabilities:
+ # drop:
+ # - ALL
+ # readOnlyRootFilesystem: true
+ # runAsNonRoot: true
+ # runAsUser: 1000
+
+ # request and limit in GB, good prod sizes are 64, 420
+ resources: {}
+ # We usually recommend not to specify default resources and to leave this as a conscious
+ # choice for the user. This also increases chances charts run on environments with little
+ # resources, such as Minikube. If you do want to specify resources, uncomment the following
+ # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+ # limits:
+ # cpu: 100m
+ # memory: 128Mi
+ # requests:
+ # cpu: 100m
+ # memory: 128Mi
+
+ # Additional volumes on the output Deployment definition.
+ volumes: []
+ # - name: foo
+ # secret:
+ # secretName: mysecret
+ # optional: false
+
+ # Additional volumeMounts on the output Deployment definition.
+ volumeMounts: []
+ # - name: foo
+ # mountPath: "/etc/foo"
+ # readOnly: true
+
+ nodeSelector: {}
+ tolerations: []
+ affinity: {}
+
+ podAnnotations: {}
+ podLabels: {}
+
## Everthing concerning the nomad gui
gui:
## This variable is used in the GUI to show or hide additional information
diff --git a/ops/kubernetes/values.yaml b/ops/kubernetes/values.yaml
index 21da8430e3..0de15f2c73 100644
--- a/ops/kubernetes/values.yaml
+++ b/ops/kubernetes/values.yaml
@@ -152,6 +152,21 @@ nomad:
requests:
memory: "8Gi"
+ adminconsole:
+ replicaCount: 1
+ nodeSelector:
+ environment: prod
+ "nomad-lab.eu/worker": ""
+ podSecurityContext:
+ runAsUser: 25249
+ runAsGroup: 11320
+ fsGroup: 11320
+ resources:
+ limits:
+ memory: "32Gi"
+ requests:
+ memory: "8Gi"
+
rabbitmq:
nodeSelector:
environment: prod
@@ -196,8 +211,8 @@ jupyterhub:
runAsGroup: 11320
fsGroup: 11320
nodeSelector:
- environment: prod
- "nomad-lab.eu/worker-north": ""
+ environment: prod
+ "nomad-lab.eu/worker-north": ""
prePuller:
hook:
nodeSelector:
--
GitLab