deploy.sh 17.5 KB
Newer Older
1
#!/bin/bash
2
nomadRoot=${nomadRoot:-/nomad/nomadlab}
3
4
buildDocker=1
updateDeploy=1
5
target_hostname=${target_hostname:-$HOSTNAME}
6
7
chownRoot=
tls=
8
debug=
9
10
alwaysPull=
secretWebCerts=
11

12
13
while test ${#} -gt 0
do
14
    case "$1" in
15
16
17
      --debug)
          debug=1
          ;;
18
19
20
      --tls)
          tls=--tls
          ;;
21
22
23
24
      --docker-only)
          buildDocker=1
          updateDeploy=""
          ;;
25
      --always-pull)
26
27
          alwaysPull=1
          ;;
28
29
30
31
      --docker-skip)
          buildDocker=""
          updateDeploy=1
          ;;
32
33
34
      --no-push)
          noPush=1
          ;;
35
36
37
38
      --secret-web-certs)
          shift
          secretWebCerts=${1:-web-certs}
          ;;
39
40
41
42
      --env)
          shift
          NODE_ENV=$1
          ;;
43
44
45
46
47
48
49
50
      --target-hostname)
          shift
          target_hostname=$1
          ;;
      --nomad-root)
          shift
          nomadRoot=$1
          ;;
51
52
53
54
      --chown-root)
          shift
          chownRoot=$1
          ;;
55
      *)
56
          echo "usage: $0 [--debug] [--tls] [--nomad-root <pathToNomadRoot>] [--chown-root <pathForPrometheusVolumes>] [--env <NODE_ENV_VALUE>] [--docker-only] [--docker-skip] [--target-hostname hostname] [--secret-web-certs <secretName>]  [--always-pull] [--no-push]"
57
          echo
58
          echo "Env variables: NODE_ENV, target_hostname, nomadRoot"
59
60
61
62
63
64
65
66
67
68
          echo "Examples:"
          echo "export NODE_ENV=nomad-vis-test"
          echo "export NODE_ENV=labenv"
          echo "export NODE_ENV=analytics-toolkit"
          exit 0
          ;;
  esac
  shift
done

69
chownRoot=${chownRoot:-$nomadRoot/servers/$target_hostname}
70
version=$(git describe --tags --always --dirty)
71
name="analytics-toolkit.nomad-coe.eu:5509/nomadlab/nomad-container-manager:$version"
72
73
if [ -n "$buildDocker" ] ; then
    docker build -t $name .
74
75
76
    if [ -z "$noPush" ] ; then
        docker push $name
    fi
77
fi
78
79
80
81
82
if [ -n "$alwaysPull" ] ; then
    pullPolicy=Always
else
    pullPolicy=IfNotPresent
fi
83

84
85
86
87
echo "# Initial setup"
echo "To make kubectl work, for example for the test kubernetes"
echo "  export KUBECONFIG=/nomad/nomadlab/kubernetes/dev/config"

88
echo "# Helm install"
89
if [ -n updateDeploy ]; then
90
    cat > helm-tiller-serviceaccount.yaml <<EOF
Mohamed, Fawzi Roberto (fawzi)'s avatar
Mohamed, Fawzi Roberto (fawzi) committed
91
apiVersion: v1
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
kind: ServiceAccount
metadata:
  name: tiller
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: tiller
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
  - kind: ServiceAccount
    name: tiller
    namespace: kube-system
EOF
Mohamed, Fawzi Roberto (fawzi)'s avatar
Mohamed, Fawzi Roberto (fawzi) committed
110
    cat > prometheus-alertmanager-volume.yaml <<EOF
111
112
113
114
115
116
117
118
119
120
apiVersion: v1
kind: PersistentVolume
metadata:
  name: prometheus-alertmanager
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
Mohamed, Fawzi Roberto (fawzi)'s avatar
Mohamed, Fawzi Roberto (fawzi) committed
121
  storageClassName: manual-alertmanager
122
  hostPath:
123
    path: $chownRoot/prometheus/alertmanager-volume
124
125
126
127
128
129
130
131
132
133
EOF

    cat > prometheus-server-volume.yaml <<EOF
apiVersion: v1
kind: PersistentVolume
metadata:
  name: prometheus-server
spec:
  capacity:
    storage: 16Gi
Mohamed, Fawzi Roberto (fawzi)'s avatar
Mohamed, Fawzi Roberto (fawzi) committed
134
  storageClassName: manual-prometheus
135
136
137
138
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  hostPath:
139
    path: $chownRoot/prometheus/server-volume
140
141
142
EOF

    cat > prometheus-values.yaml <<EOF
Mohamed, Fawzi Roberto (fawzi)'s avatar
Mohamed, Fawzi Roberto (fawzi) committed
143
144
145
146
147
148
149
150
151
152
alertmanager:
  persistentVolume:
    storageClass: manual-alertmanager
  service:
    type: NodePort
server:
  persistentVolume:
    storageClass: manual-prometheus
  service:
    type: NodePort
153
154
155
EOF
fi

Mohamed, Fawzi Roberto (fawzi)'s avatar
Mohamed, Fawzi Roberto (fawzi) committed
156
echo "  kubectl create -f helm-tiller-serviceaccount.yaml"
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
if [ -n "$tls" ] ; then
    echo "# secure heml as described in https://docs.helm.sh/using_helm/#using-ssl-between-helm-and-tiller"
    echo "# create certificates"
    echo "mkdir helm-certs"
    echo "cd helm-certs"
    echo "openssl genrsa -out ./ca.key.pem 4096"
    echo "openssl req -key ca.key.pem -new -x509 -days 7300 -sha256 -out ca.cert.pem -extensions v3_ca"
    echo "openssl genrsa -out ./tiller.key.pem 4096"
    echo "openssl genrsa -out ./helm.key.pem 4096"
    echo "openssl req -key tiller.key.pem -new -sha256 -out tiller.csr.pem"
    echo "openssl req -key helm.key.pem -new -sha256 -out helm.csr.pem"
    echo "openssl x509 -req -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -in tiller.csr.pem -out tiller.cert.pem -days 365"
    echo "openssl x509 -req -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -in helm.csr.pem -out helm.cert.pem  -days 365"
    echo "cp ca.cert.pem \$(helm home)/ca.pem"
    echo "cp helm.cert.pem \$(helm home)/cert.pem"
    echo "cp helm.key.pem \$(helm home)/key.pem"
    echo "# initialize helm"
    echo "helm init --override 'spec.template.spec.containers[0].command'='{/tiller,--storage=secret}' \\"
    echo "          --tiller-tls \\"
    echo "          --tiller-tls-verify \\"
    echo "          --tiller-tls-cert=cert.pem \\"
    echo "          --tiller-tls-key=key.pem \\"
    echo "          --tls-ca-cert=ca.pem \\"
    echo "          --service-account=tiller"
else
    echo "  helm init --service-account tiller"
fi
184
185
186
echo "# Prometheus setup"
echo "  kubectl create -f prometheus-alertmanager-volume.yaml"
echo "  kubectl create -f prometheus-server-volume.yaml"
187
echo "  helm install $tls --name prometheus -f prometheus-values.yaml stable/prometheus"
188
189
190

if [ -n updateDeploy ]; then
    cat >container-manager-namespace.yaml <<EOF
191
192
193
194
195
kind: Namespace
apiVersion: v1
metadata:
  name: analytics
EOF
196
fi
197

198
199
200
201
202
203
204
echo "## Environment setup, redis db for the sessions"
if [ -n updateDeploy ]; then
    if [ ! -e session-db-redis-pwd.txt ]; then
        echo "created random password in session-db-redis-pwd.txt"
        cat /dev/urandom | tr -dc 'a-zA-Z0-9' | head -c 28 > session-db-redis-pwd.txt
    fi

Mohamed, Fawzi Roberto (fawzi)'s avatar
Mohamed, Fawzi Roberto (fawzi) committed
205
    SESSION_REDIS_PASS=$(head -1 session-db-redis-pwd.txt)
206
207

    cat >session-redis-helm-values.yaml <<EOF
208
209
210
image:
  pullPolicy: $pullPolicy
existingSecret: analytics-session-db
211
212
213
214
215
rbac:
  create: true
metrics:
  enabled: true
master:
216
217
  service:
    type: NodePort
218
  disableCommands: ""
219
220
221
222
  persistence:
    enabled: false
cluster:
  enabled: false
223
EOF
224
fi
225
echo "# password secret"
226
echo "  kubectl create secret generic analytics-session-db --from-file=redis-password=session-db-redis-pwd.txt"
227
echo "# actual redis setup"
228
229
echo "  if ! [[ -n \"\$(helm ls $tls analytics-session-db | grep -E '^analytics-session-db\s' )\" ]]; then"
echo "    helm install $tls --name analytics-session-db -f session-redis-helm-values.yaml stable/redis"
230
echo "  else"
231
echo "    helm upgrade $tls analytics-session-db -f session-redis-helm-values.yaml stable/redis"
232
echo "  fi"
233
234


235
echo "## Environment setup, mongo db for notebook & usage information"
236
if [ -n updateDeploy ]; then
237
238
239
240
241

    if [ ! -e notebook-db-mongo-pwd.txt ]; then
        echo "created random password in notebook-db-mongo-pwd.txt"
        cat /dev/urandom | tr -dc 'a-zA-Z0-9' | head -c 28 > notebook-db-mongo-pwd.txt
    fi
Mohamed, Fawzi Roberto (fawzi)'s avatar
Mohamed, Fawzi Roberto (fawzi) committed
242
243
    NOTEBOOK_MONGO_PASS=$(head -1 notebook-db-mongo-pwd.txt)

244
245
# check how to use secret
    cat >notebook-mongo-helm-values.yaml <<EOF
246
247
image:
  pullPolicy: $pullPolicy
248
mongodbRooPassword: "$(cat notebook-db-mongo-pwd.txt)"
249
mongodbUsername: "notebookinfo"
250
mongodbPassword: "$(cat notebook-db-mongo-pwd.txt)"
251
mongodbDatabase: "notebookinfo"
Mohamed, Fawzi Roberto (fawzi)'s avatar
Mohamed, Fawzi Roberto (fawzi) committed
252
253
persistence:
  enabled: false
254
EOF
255
fi
256

257
258
259
echo "# password secret"
echo "  kubectl create secret generic notebook-db-mongo-pwd --from-literal=database=notebookinfo --from-literal=user=notebookinfo --from-file=password=notebook-db-mongo-pwd.txt"
echo "# actual mongo setup"
Mohamed, Fawzi Roberto (fawzi)'s avatar
Mohamed, Fawzi Roberto (fawzi) committed
260

261
echo "  if ! [[ -n \"\$(helm ls $tls notebook-info-db | grep -E '^notebook-info-db\s' )\" ]]; then"
262
echo "    helm install $tls --name notebook-info-db -f notebook-mongo-helm-values.yaml stable/mongodb"
Mohamed, Fawzi Roberto (fawzi)'s avatar
Mohamed, Fawzi Roberto (fawzi) committed
263
echo "  else"
264
echo "    helm upgrade $tls notebook-info-db -f notebook-mongo-helm-values.yaml stable/mongodb"
Mohamed, Fawzi Roberto (fawzi)'s avatar
Mohamed, Fawzi Roberto (fawzi) committed
265
echo "  fi"
266

267
echo "## Environment setup: user settings redis db"
268
if [ -n updateDeploy ]; then
269
270
271
272
273
274
    if [ ! -e user-settings-redis-pwd.txt ]; then
        echo "created random password in user-settings-redis-pwd.txt"
        cat /dev/urandom | tr -dc 'a-zA-Z0-9' | head -c 28 > user-settings-redis-pwd.txt
    fi

    cat >user-settings-redis-volume.yaml <<EOF
275
apiVersion: v1
276
kind: PersistentVolume
277
metadata:
278
  name: user-settings-redis
279
280
281
  labels:
    type: local
spec:
Mohamed, Fawzi Roberto (fawzi)'s avatar
Mohamed, Fawzi Roberto (fawzi) committed
282
  storageClassName: manual-user-settings
283
  capacity:
284
    storage: 16Gi
285
286
  accessModes:
    - ReadWriteOnce
287
  persistentVolumeReclaimPolicy: Recycle
288
  hostPath:
289
290
    path: $nomadRoot/servers/$target_hostname/analytics/user-settings-redis-data
    type: Directory
291
292
EOF

293
    cat >user-settings-redis-helm-values.yaml <<EOF
294
295
image:
  pullPolicy: $pullPolicy
296
297
298
299
300
301
existingSecret: user-settings-db
rbac:
  create: true
metrics:
  enabled: true
master:
302
303
  service:
    type: NodePort
304
  disableCommands: ""
305
306
  persistence:
    enabled: true
Mohamed, Fawzi Roberto (fawzi)'s avatar
Mohamed, Fawzi Roberto (fawzi) committed
307
    storageClass: manual-user-settings
308
309
cluster:
  enabled: false
310
EOF
311
fi
312
313


314
315
316
317
318
echo "# volume for redis persistence"
echo "  kubectl create -f user-settings-redis-volume.yaml"
echo "# password secret"
echo "  kubectl create secret generic user-settings-db --from-file=redis-password=user-settings-redis-pwd.txt"
echo "# actual redis setup"
319
echo "  if ! [[ -n \"\$(helm ls $tls user-settings-db | grep -E '^user-settings-db\s' )\" ]]; then"
320
echo "    helm install $tls --name user-settings-db -f user-settings-redis-helm-values.yaml stable/redis"
321
echo "  else"
322
echo "    helm upgrade $tls user-settings-db -f user-settings-redis-helm-values.yaml stable/redis"
323
324
echo "  fi"

325
echo "## Environment setup for container manager"
326
if [ -n updateDeploy ]; then
327
328
329
330
331
332
cat >container-manager-namespace.yaml <<EOF
kind: Namespace
apiVersion: v1
metadata:
  name: analytics
EOF
333
fi
334
echo "# create namespace for pods of container manager"
335
echo "  kubectl create -f container-manager-namespace.yaml"
336
337
echo "# secret to pull images from analytics-toolkit.nomad-coe.eu:5509"
echo "  kubectl create secret docker-registry garching-kube --docker-server=analytics-toolkit.nomad-coe.eu:5509 --docker-username=\$DOCKER_USERNAME --docker-password=\"\$DOCKER_PASSWORD\" --docker-email=\$DOCKER_EMAIL"
338
echo "  kubectl --namespace analytics create secret docker-registry garching-kube --docker-server=analytics-toolkit.nomad-coe.eu:5509 --docker-username=\$DOCKER_USERNAME --docker-password=\"\$DOCKER_PASSWORD\" --docker-email=\$DOCKER_EMAIL"
339
340
341
echo "# get certificates to connect to kubernetes (either from kube-certs of ~/.minikube)"
echo "  if [ -e kube-certs ] ; then"
echo "    pushd kube-certs"
342
343
echo "  elif [ -e ../kube-certs ] ; then"
echo "    pushd ../kube-certs"
344
echo "  elif [ -e ~/.minikube ] ; then"
345
346
347
348
349
350
351
352
353
354
355
356
357
358
echo "    mkdir -p ./kube-certs"
echo "    pushd  kube-certs"
echo "    cp ~/.minikube/ca.crt ~/.minikube/client.crt ~/.minikube/client.key ."
echo "    sed \"s|$HOME/.minikube|/usr/src/app/kube-certs|g\" ~/.kube/config > config"
if [ -z "$KUBERNETES_SERVER_URL" ] ; then
    echo "    echo https://\$(minikube ip):8443 > server.url"
else
    echo "    echo \"$KUBERNETES_SERVER_URL\" > server.url"
fi
if [ -z "$KUBERNETES_NODE" ] ; then
    echo "    echo \$(minikube ip) >node.addr"
else
    echo "    echo \"$KUBERNETES_NODE\" > node.addr"
fi
359
360
361
echo "  else"
echo "    pushd ."
echo "  fi"
362
363

if [ -n "$KUBERNETES_SERVER_URL" ]; then
364
    echo "  kubectl create secret generic kube-certs --from-file=ca.crt=ca.crt --from-file=client.crt=client.crt --from-file=client.key=client.key --from-literal=server.url=\"$KUBERNETES_SERVER_URL\" --from-literal=node.addr=\"$KUBERNETES_NODE\" --from-file=config=config"
365
else
366
    echo "  kubectl create secret generic kube-certs --from-file=ca.crt=ca.crt --from-file=client.crt=client.crt --from-file=client.key=client.key --from-file=server.url=server.url --from-file=node.addr=node.addr --from-file=config=config"
367
368
369
370
371
372
373
374
375
376
377
378
379
fi
echo "  popd"
echo "# create secret with web certificates"
echo "  if [ -e web-certs ] ; then"
echo "    pushd web-certs"
echo "  elif [ -e ../web-certs ]; then"
echo "    pushd ../web-certs"
echo "  else"
echo "    pushd ."
echo "  fi"
echo "  kubectl create secret generic ${secretWebCerts} --from-file=key.pem=key.pem --from-file=cert.pem=cert.pem"
echo "  popd"
echo
380

381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
echo "# create cron job updating info on services"
if [ -n updateDeploy ]; then
targetF=service-dumper.yaml
cat > "$targetF" <<EOF
apiVersion: batch/v1beta1
kind: CronJob
metadata:
  name: hello
spec:
  schedule: "*/15 * * * *"
  jobTemplate:
    spec:
      template:
        spec:
          metadata:
            labels:
              app: nomad-container-manager-dumper
          imagePullSecrets:
          - name: garching-kube
          restartPolicy: OnFailure
          containers:
          - name: dumper
            image: $name
            imagePullPolicy: $pullPolicy
            command:
            - node
            - app.js
            - serviceDumper
            - --out-file
            - /services-info/$target_hostname.services.yaml
            env:
            - name: KUBERNETES_SERVER_URL
              valueFrom:
                secretKeyRef:
                  name: kube-certs
                  key: server.url
            - name: KUBERNETES_NODE
              valueFrom:
                secretKeyRef:
                  name: kube-certs
                  key: node.addr
            - name: NODE_ENV
              value: "$NODE_ENV"
            - name: NODE_APP_INSTANCE
              value: "$imageType"
            volumeMounts:
            - mountPath: "/services-info"
              name:  services-info
EOF
    if [ -n "$debug" ] ; then
        cat >> $targetF <<EOF
            - mountPath: "/usr/src/app"
              name: app-source
EOF
    fi
    cat >> $targetF <<EOF
            - mountPath: "/usr/src/app/kube-certs"
              name: kube-certs
              readOnly: true
          volumes:
          - name: kube-certs
            secret:
              secretName: kube-certs
          - name: services-info
            hostPath:
              path: "$nomadRoot/servers/$target_hostname/analytics/services-info"
EOF
    if [ -n "$debug" ] ; then
        cat >> $targetF <<EOF
          - name: app-source
            hostPath:
              path: "$nomadRoot/servers/$target_hostname/analytics/$imageType"
EOF
    fi
fi
456
echo "  kubectl create -f $targetF"
457

458
for imageType in beaker jupyter creedo remotevis ; do
459

460
echo "## $imageType Initial setup: create container manager service"
461
if [ -n updateDeploy ]; then
462
cat >container-manager-service-$imageType.yaml <<HERE
463
464
465
kind: Service
apiVersion: v1
metadata:
466
  name: nomad-container-manager-$imageType
467
468
469
spec:
  selector:
    app: nomad-container-manager
470
    imageType: $imageType
471
472
  ports:
  - protocol: TCP
473
474
    port: 3003
    targetPort: 3003
475
476
  type: NodePort
HERE
477
fi
478
echo "  kubectl create -f container-manager-service-$imageType.yaml"
479

480
481
482
if [ -n "$updateDeploy" ]; then
    targetF=container-manager-deploy-$imageType.yaml
    cat >$targetF <<HERE
483
484
485
apiVersion: apps/v1beta2
kind: Deployment
metadata:
486
  name: nomad-container-manager-$imageType
487
488
  labels:
    app: nomad-container-manager
489
    imageType: $imageType
490
491
492
493
494
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nomad-container-manager
495
      imageType: $imageType
496
497
498
499
  template:
    metadata:
      labels:
        app: nomad-container-manager
500
        imageType: $imageType
501
    spec:
502
503
      imagePullSecrets:
      - name: garching-kube
504
505
506
      containers:
      - name: nomad-container-manager
        image: $name
507
        imagePullPolicy: $pullPolicy
508
509
510
511
        command:
        - npm
        - start
        - webserver
512
        ports:
513
514
515
        - containerPort: 3003
        env:
        - name: SESSION_DB_PASSWORD
516
517
518
519
520
          valueFrom:
            secretKeyRef:
              name: analytics-session-db
              key: redis-password
        - name: USER_SETTINGS_DB_PASSWORD
521
522
523
524
          valueFrom:
            secretKeyRef:
              name: user-settings-db
              key: redis-password
525
        - name: NOTEBOOK_INFO_DB_PASSWORD
526
527
528
529
          valueFrom:
            secretKeyRef:
              name: notebook-db-mongo-pwd
              key: password
530
        - name: KUBERNETES_SERVER_URL
531
532
533
534
          valueFrom:
            secretKeyRef:
              name: kube-certs
              key: server.url
535
        - name: KUBERNETES_NODE
536
537
538
539
          valueFrom:
            secretKeyRef:
              name: kube-certs
              key: node.addr
540
        - name: NODE_ENV
541
          value: "$NODE_ENV"
542
543
544
        - name: NODE_APP_INSTANCE
          value: "$imageType"
        volumeMounts:
545
546
547
HERE
    if [ -n "$debug" ] ; then
        cat >> $targetF <<EOF
548
        - mountPath: "/usr/src/app"
549
          name: app-source
550
551
552
553
554
555
556
557
558
EOF
    fi
    if [ -n "$secretWebCerts" ] ; then
        cat >> $targetF <<EOF
        - mountPath: "/usr/src/app/web-certs"
          name: web-certs
EOF
    fi
    cat >> $targetF <<EOF
559
560
561
        - mountPath: "/usr/src/app/kube-certs"
          name: kube-certs
          readOnly: true
562
        - mountPath: "/nomad/nomadlab/user-data/shared"
563
          name: user-shared
564
        - mountPath: "/nomad/nomadlab/user-data/private"
565
          name: user-private
566
      volumes:
567
568
569
      - name: kube-certs
        secret:
          secretName: kube-certs
570
571
572
573
574
575
      - name: user-shared
        hostPath:
          path: $nomadRoot/user-data/shared
      - name: user-private
        hostPath:
          path: $nomadRoot/user-data/private
576
577
578
EOF
    if [ -n "$debug" ] ; then
        cat >> $targetF <<EOF
579
580
      - name: app-source
        hostPath:
581
          path: "$nomadRoot/servers/$target_hostname/analytics/$imageType"
582
EOF
583
584
585
586
587
588
589
590
    fi
    if [ -n "$secretWebCerts" ] ; then
        cat >> $targetF <<EOF
      - name: web-certs
        secret:
          secretName: $secretWebCerts
EOF
    fi
591
fi
592

593
594
595
596
597
598
599
echo "if ! kubectl get deployment nomad-container-manager-$imageType >& /dev/null ;  then"
echo "  kubectl create --save-config -f container-manager-deploy-$imageType.yaml"
echo "else"
echo "  kubectl apply -f container-manager-deploy-$imageType.yaml"
echo "fi"
echo
done