Commit 0ad09686 authored by Mohamed, Fawzi Roberto (fawzi)'s avatar Mohamed, Fawzi Roberto (fawzi)
Browse files

first general deploy doc

parent cdb90493
Pipeline #39079 failed with stages
in 47 minutes and 23 seconds
# NOMAD deployments
This in information on how to deploy the archive and analytics toolkit.
Some information is a bit specific to our own production and development systems.
In general scripts create files to do the installation and print out commands.
For security reasons they do not execute the deployment themselves.
You might want to do it manually, or deploy only part of the system.
## deploy/kubernetes
## deploy/base
contains info for deploying the basic infrastructure on the top of kubernetes (execute the baseSetup.sh script)
## deploy/frontend
## deploy/api
## deploy/container-manager
## Machine specific hints:
#!/bin/bash
nomadRoot=${nomadRoot:-/nomad/nomadlab}
updateDeploy=1
target_hostname=${target_hostname:-$HOSTNAME}
chownRoot=
tls=
secretWebCerts=
while test ${#} -gt 0
do
case "$1" in
--tls)
tls=--tls
;;
--secret-web-certs)
shift
secretWebCerts=${1:-web-certs}
;;
--target-hostname)
shift
target_hostname=$1
;;
--nomad-root)
shift
nomadRoot=$1
;;
--chown-root)
shift
chownRoot=$1
;;
*)
echo "usage: $0 [--tls] [--nomad-root <pathToNomadRoot>] [--chown-root <pathForPrometheusVolumes>] [--target-hostname hostname]"
echo
echo "Env variables: target_hostname, nomadRoot"
exit 0
;;
esac
shift
done
chownRoot=${chownRoot:-$nomadRoot/servers/$target_hostname}
echo "# Initial setup"
echo "To make kubectl work, for example for the test kubernetes"
echo " export KUBECONFIG=/etc/kubernetes/admin.conf"
echo "# Helm install"
if [ -n updateDeploy ]; then
cat > helm-tiller-serviceaccount.yaml <<EOF
apiVersion: v1
kind: ServiceAccount
metadata:
name: tiller
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: tiller
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: tiller
namespace: kube-system
EOF
cat > prometheus-alertmanager-volume.yaml <<EOF
apiVersion: v1
kind: PersistentVolume
metadata:
name: prometheus-alertmanager
spec:
capacity:
storage: 5Gi
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Recycle
storageClassName: manual-alertmanager
hostPath:
path: $chownRoot/prometheus/alertmanager-volume
EOF
cat > prometheus-server-volume.yaml <<EOF
apiVersion: v1
kind: PersistentVolume
metadata:
name: prometheus-server
spec:
capacity:
storage: 16Gi
storageClassName: manual-prometheus
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Recycle
hostPath:
path: $chownRoot/prometheus/server-volume
EOF
cat > prometheus-values.yaml <<EOF
alertmanager:
persistentVolume:
storageClass: manual-alertmanager
service:
type: NodePort
server:
persistentVolume:
storageClass: manual-prometheus
service:
type: NodePort
EOF
fi
echo " kubectl create -f helm-tiller-serviceaccount.yaml"
if [ -n "$tls" ] ; then
echo "# secure heml as described in https://docs.helm.sh/using_helm/#using-ssl-between-helm-and-tiller"
echo "# create certificates"
echo "mkdir helm-certs"
echo "cd helm-certs"
echo "openssl genrsa -out ./ca.key.pem 4096"
echo "openssl req -key ca.key.pem -new -x509 -days 7300 -sha256 -out ca.cert.pem -extensions v3_ca"
echo "openssl genrsa -out ./tiller.key.pem 4096"
echo "openssl genrsa -out ./helm.key.pem 4096"
echo "openssl req -key tiller.key.pem -new -sha256 -out tiller.csr.pem"
echo "openssl req -key helm.key.pem -new -sha256 -out helm.csr.pem"
echo "openssl x509 -req -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -in tiller.csr.pem -out tiller.cert.pem -days 365"
echo "openssl x509 -req -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -in helm.csr.pem -out helm.cert.pem -days 365"
echo "cp ca.cert.pem \$(helm home)/ca.pem"
echo "cp helm.cert.pem \$(helm home)/cert.pem"
echo "cp helm.key.pem \$(helm home)/key.pem"
echo "# initialize helm"
echo "helm init --override 'spec.template.spec.containers[0].command'='{/tiller,--storage=secret}' \\"
echo " --tiller-tls \\"
echo " --tiller-tls-verify \\"
echo " --tiller-tls-cert=cert.pem \\"
echo " --tiller-tls-key=key.pem \\"
echo " --tls-ca-cert=ca.pem \\"
echo " --service-account=tiller"
else
echo " helm init --service-account tiller"
fi
echo "# Prometheus setup"
echo " kubectl create -f prometheus-alertmanager-volume.yaml"
echo " kubectl create -f prometheus-server-volume.yaml"
echo " helm install $tls --name prometheus -f prometheus-values.yaml stable/prometheus"
from labdev-nomad.container
cd /nomad/nomadlab/servers/nomad-vis-test/analytics/remotevis
Update info on services of labdev that we use as we share the session db (we should probably clean up this ugly command)
kubectl exec -ti $(kubectl get po | grep nomad-container-manager-beaker | cut -f1 -d ' ') node app.js serviceDumper -- --out-file labdev-nomad.services.yaml
update config with current info on the redis dbs of labdev (default-remotevis.hjson.in -> default-remotevis.hjson)
docker run -ti -v $PWD:/usr/src/app -v /nomad/nomadlab/servers/labdev-nomad/analytics/beaker:/mnt -w /usr/src/app --rm node:carbon node app.js templateEvaluer --replacements /mnt/labdev-nomad.services.yaml --template config/nomad-vis-test.hjson.in --out-file config/nomad-vis-test.hjson
deploy
./deploy.sh --tls --env nomad-vis-test --target-hostname nomad-vis-test --secret-web-certs web-certs
and execute the deploy for remote vis
kubectl create -f container-manager-service-remotevis.yaml
if ! kubectl get deployment nomad-container-manager-remotevis >& /dev/null ; then
kubectl create --save-config -f container-manager-deploy-remotevis.yaml
else
kubectl apply -f container-manager-deploy-remotevis.yaml
fi
if only that changed, otherwise on has also to create the secrets and analytics namespace.
A serviceDump has to be run to reexport the ports to the frontend, then the frontend setup needs to be updated.
---
frontend:
- server_name: labdev-nomad.esc.rzg.mpg.de
ssl_certificate: /web-certs/cert.pem
ssl_certificate_key: /web-certs/key.pem
shortcuts: true
repoapi:
- nodes:
- staging-nomad.esc.rzg.mpg.de
ports:
- nodePort: 8111
industry-project-imeall:
- nodes:
- labdev-nomad.esc.rzg.mpg.de
ports:
- nodePort: 34695
---
frontend:
server_name: analytics-toolkit.nomad-coe.eu
other_servers: |
server {
listen 80;
server_name labtest-nomad.esc.rzg.mpg.de;
return 301 https://$server_name$request_uri;
}
server {
listen 443 ssl;
server_name labtest-nomad.esc.rzg.mpg.de;
ssl_certificate /certs/cert-8701391933287641330712620431.pem;
ssl_certificate_key /certs/labtest-nomad.esc.rzg.mpg.de.key;
return 301 https://analytics-toolkit.nomad-coe.eu/$request_uri;
}
shortcuts: true
user nginx;
worker_processes 1;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;
events {
worker_connections 1024;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
server_names_hash_bucket_size 64;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
#tcp_nopush on;
#gzip on;
tcp_nopush on;
# i tried setting to 0 and removing the keepalive timer from the ipython client.
# but it did not fix the problem.
#keepalive_timeout 0;
keepalive_timeout 1000;
proxy_connect_timeout 90;
proxy_send_timeout 90;
proxy_read_timeout 1000000;
proxy_buffers 32 4k;
client_max_body_size 100M;
client_body_buffer_size 128k;
client_body_temp_path "client_temp";
proxy_temp_path "client_temp";
fastcgi_temp_path "client_temp";
uwsgi_temp_path "client_temp";
scgi_temp_path "client_temp";
server {
listen 80;
server_name labtest-nomad.esc.rzg.mpg.de;
return 301 https://$server_name$request_uri;
}
server {
listen 443 ssl;
server_name labtest-nomad.esc.rzg.mpg.de;
ssl_certificate /certs/cert-8701391933287641330712620431.pem;
ssl_certificate_key /certs/labtest-nomad.esc.rzg.mpg.de.key;
return 301 https://analytics-toolkit.nomad-coe.eu/$request_uri;
}
server {
listen 80;
server_name analytics-toolkit.nomad-coe.eu;
return 301 https://$server_name$request_uri;
}
server {
autoindex off;
# enable requests for specific instances in multi-user environments
# where a request could be routed to one of many server instances
# /beaker/<uuid>/foo -> /foo
rewrite "^/beaker/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/(.*)$" /b7c81a9/$1 last;
# listen 0.0.0.0:80 ssl;
listen 5509 ssl;
server_name analytics-toolkit.nomad-coe.eu;
#server_name 7741588557007104
ssl_certificate /etc/certs/nomad-coe.eu.chain.pem;
ssl_certificate_key /etc/certs/nomad-coe.eu.key.pem;
#ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
#ssl_ciphers HIGH:!aNULL:!MD5;
# allow large uploads of files
client_max_body_size 3000m; #1G;
location / {
#proxy_http_version 1.1;
# optimize downloading files larger than 1G
proxy_max_temp_file_size 3000m;
proxy_buffering off;
chunked_transfer_encoding on;
proxy_request_buffering off;
proxy_pass http://nomad-toolkit-prod2.esc.rzg.mpg.de:30170;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto "https";
}
}
server {
autoindex off;
# enable requests for specific instances in multi-user environments
# where a request could be routed to one of many server instances
# /beaker/<uuid>/foo -> /foo
rewrite "^/beaker/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/(.*)$" /b7c81a9/$1 last;
# listen 0.0.0.0:80 ssl;
listen 443 ssl;
server_name analytics-toolkit.nomad-coe.eu;
#server_name 7741588557007104
ssl_certificate /etc/certs/nomad-coe.eu.chain.pem;
ssl_certificate_key /etc/certs/nomad-coe.eu.key.pem;
#ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
#ssl_ciphers HIGH:!aNULL:!MD5;
# allow large uploads of files
client_max_body_size 1G;
# optimize downloading files larger than 1G
#proxy_max_temp_file_size 2G;
# redirect server error pages to the static page /50x.html and serve them directly from static html directory
error_page 500 502 503 504 /static/50x.html;
location /personal/ {
alias "/usr/share/nginx/html/personal/";
try_files $uri $uri/ /personal/index.html;
}
# login and loginrest are used for the public server option
location = /login {
proxy_set_header Host analytics-toolkit.nomad-coe.eu;
# proxy_set_header Origin https://130.183.207.113:31548;
proxy_set_header Origin "https://130.183.207.103:8801";
# proxy_pass https://130.183.207.113:31548;
proxy_pass https://130.183.207.103:8801;
}
location /stats {
proxy_pass http://130.183.207.112:31393;
# proxy_pass http://labdev3-nomad.esc.rzg.mpg.de:30403;
# proxy_pass http://130.183.207.103:3838;
}
location /stats-meta {
proxy_pass http://130.183.207.103:3838;
}
location = /Shibboleth.sso {
# proxy_pass https://130.183.207.113:31548;
proxy_pass https://130.183.207.103:8801;
}
location /Shibboleth.sso/ {
# proxy_pass https://130.183.207.113:31548;
proxy_pass https://130.183.207.103:8801;
}
location = /shibboleth {
# proxy_pass https://130.183.207.113:31548;
proxy_pass https://130.183.207.103:8801;
}
location /shibboleth/ {
# proxy_pass https://130.183.207.113:31548;
proxy_pass https://130.183.207.103:8801;
}
location /login/ {
proxy_set_header Host analytics-toolkit.nomad-coe.eu;
# proxy_set_header Origin "https://130.183.207.113:31548";
proxy_set_header Origin "https://130.183.207.103:8801";
# proxy_pass https://130.183.207.113:31548;
proxy_pass https://130.183.207.103:8801;
}
# version get request
location /notebook-edit/ {
# proxy_pass https://130.183.207.113:31548;
proxy_pass https://130.183.207.103:8801;
}
location /userapi/ {
proxy_pass https://130.183.207.103:8802;
}
location /api/ {
proxy_pass http://nomad-flink-01.esc.rzg.mpg.de:30050;
# proxy_pass http://labdev4-nomad.esc.rzg.mpg.de:32728;
# proxy_pass http://130.183.207.112:31393;
# proxy_pass http://130.183.207.100:31235;
}
location /analytics/ {
proxy_pass http://labdev3-nomad.esc.rzg.mpg.de:32187;
}
# location /archive/nql-api/simple_stats {
# proxy_pass http://130.183.207.112:31596;
# }
location /archive/ {
# proxy_pass http://labdev3-nomad.esc.rzg.mpg.de:31263;
# proxy_pass http://130.183.207.112:30889;
proxy_pass http://130.183.207.112:31596;
}
location /ui/ {
proxy_pass http://nomad-flink-01.esc.rzg.mpg.de:30050;
# proxy_pass http://labdev3-nomad.esc.rzg.mpg.de:32728;
# proxy_pass http://130.183.207.112:31393;
}
location /nmi/ {
proxy_pass http://labdev3-nomad.esc.rzg.mpg.de:32728;
# proxy_pass http://130.183.207.112:31393;
}
#Add jupyter notebooks
location /jupyter/ {
proxy_set_header Host labtest-nomad.esc.rzg.mpg.de;
#proxy_pass https://labtest-nomad.esc.rzg.mpg.de:8807;
proxy_pass https://nomad-flink-01.esc.rzg.mpg.de:32141;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto "https";
#proxy_redirect off;
#proxy_buffering off;
#proxy_set_header Upgrade $http_upgrade;
#proxy_set_header Connection "Upgrade";
#proxy_read_timeout 86400;
#proxy_http_version 1.1;
}
location = /jupyter {
proxy_set_header Host labtest-nomad.esc.rzg.mpg.de;
proxy_pass https://nomad-flink-01.esc.rzg.mpg.de:32141;
#proxy_pass https://labtest-nomad.esc.rzg.mpg.de:8807;
}
location ~ /jupyter/api/kernels/ {
proxy_pass https://nomad-flink-01.esc.rzg.mpg.de:32141;
# proxy_pass https://labtest-nomad.esc.rzg.mpg.de:8807;
proxy_set_header Host $host;
# websocket support
proxy_http_version 1.1;
proxy_set_header Upgrade "WebSocket";
proxy_set_header Connection "Upgrade";
proxy_read_timeout 86400;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto "https";
}
location ~ /jupyter/terminals/ {
proxy_pass https://nomad-flink-01.esc.rzg.mpg.de:32141;
#proxy_pass https://labtest-nomad.esc.rzg.mpg.de:8807;
proxy_set_header Host $host;
# websocket support
proxy_http_version 1.1;
proxy_set_header Upgrade "WebSocket";
proxy_set_header Connection "Upgrade";
proxy_read_timeout 86400;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto "https";
#proxy_set_header X-NginX-Proxy true;
}
###
# redirect to the starting page
location = / {
return 301 $scheme://$http_host/home/;
}
location = /beaker/#/control {
return 301 $scheme://$http_host/home/;
}
location /nexus/ {
proxy_pass http://nomad-toolkit-prod2.esc.rzg.mpg.de:31629;
proxy_buffering off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto "https";
}
location = /Creedo {
return 301 $scheme://$http_host/Creedo/;
}
location /Creedo/ {
proxy_pass https://130.183.207.103:8805;
}
location = /zeppelin {
return 301 $scheme://$http_host/zeppelin/;
}
location /zeppelin/ {
proxy_pass http://130.183.207.103:8811;
}
location /zeppelin/ws {
proxy_pass http://130.183.207.103:8811;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
add_header Access-Control-Allow-Origin *;
}
location = /beaker {
return 301 $scheme://$http_host/beaker/;
}
location /static/ {
alias "/usr/share/nginx/html/static/";
}
location /archive-browser/ {
try_files $uri /archive-browser/index.html;
alias "/usr/share/nginx/html/archive-browser/";
}
location /analytics/springer {
alias "/usr/share/nginx/html/analytics/springer";
}
location /userapi/demos {
index index.json;
alias "/usr/share/nginx/html/userapi/demos";
}
location /home/ {
alias "/usr/share/nginx/html/home/";
}
location /.well-known/ {
alias "/usr/share/nginx/html/.well-known/";
}
location = /robots.txt {
alias "/usr/share/nginx/html/static/robots.txt";
}
location = /beaker/ {
#auth_basic "closed site";
#auth_basic_user_file /etc/nginx/htpasswd;
#proxy_pass https://130.183.207.113:31548;
proxy_pass https://130.183.207.103:8801;
}
location = /nomad-query-gui {
return 302 https://$server_name/notebook-edit/data/shared/sommerregen/nomad-query/nomad-query.bkr;
}
location = /tutorial-LASSO-L0 {
return 302 https://$server_name/notebook-edit/data/shared/tutorials/LASSO_L0.bkr;
}
location = /tutorial-metal-nonmetal {
return 302 https://$server_name/notebook-edit/data/shared/tutorialsNew/sisso/sisso-metal-nonmetal.bkr;
}
location = /tutorial-LASSO_L0 {
return 302 https://$server_name/notebook-edit/data/shared/tutorials/LASSO_L0.bkr;
}
location = /tutorial-embedding {
return 302 https://$server_name/notebook-edit/data/shared/tutorials/Embedding.bkr;
}
location = /tutorial-SGD {
return 302 https://$server_name/Creedo;
}
location = /tutorial-glosim {
return 302 https://$server_name/notebook-edit/data/shared/tutorials/SOAP_similarity.bkr;
}
location = /tutorial-krr {
return 302 https://$server_name/notebook-edit/data/shared/tutorials/brprototype3.bkr;
}
location = /tutorial-query {