Added basic liveness/readiness for api+worker

nomad/api/app.py

@@ -82,6 +82,12 @@ def handle(error: Exception):
     return response
 
 
+@app.route("%s/alive" % base_path)
+def alive():
+    """ Simply endpoint to utilize kubernetes liveness/readiness probing. """
+    return "I am, alive!"
+
+
 def with_logger(func):
     """
     Decorator for endpoint implementations that provides a pre configured logger and

ops/helm/nomad/templates/api-deployment.yaml

@@ -71,6 +71,18 @@ spec:
         - name: NOMAD_COE_REPO_DB_NAME
           value: "{{ .Values.dbname }}"
         command: ["python", "-m", "gunicorn.app.wsgiapp", "-b 0.0.0.0:8000", "nomad.api:app"]
+        livenessProbe:
+          httpGet:
+            path: "{{ .Values.proxy.external.path }}/alive"
+            port: 8000
+          initialDelaySeconds: 15
+          periodSeconds: 15
+        readinessProbe:
+          httpGet:
+            path: "{{ .Values.proxy.external.path }}/alive"
+            port: 8000
+          initialDelaySeconds: 5
+          periodSeconds: 5
       imagePullSecrets:
       - name: {{ .Values.images.secret }}
       imagePullPolicy: always

ops/helm/nomad/templates/worker-deployment.yaml

@@ -59,6 +59,26 @@ spec:
         - name: NOMAD_COE_REPO_DB_NAME
           value: "{{ .Values.dbname }}"
         command: ["python", "-m", "celery", "worker", "-l", "info", "-A", "nomad.processing"]
+        livenessProbe:
+          exec:
+            env:
+            - name: NOMAD_LOGSTASH_LEVEL
+            - value: 30
+            command:
+            - sh -c
+            - celery -A nomad.processing status
+          initialDelaySeconds: 30
+          periodSeconds: 30
+        readinessProbe:
+          exec:
+            env:
+            - name: NOMAD_LOGSTASH_LEVEL
+            - value: 30
+            command:
+            - sh -c
+            - celery -A nomad.processing status
+          initialDelaySeconds: 5
+          periodSeconds: 5
       imagePullSecrets:
       - name: {{ .Values.images.secret }}
       imagePullPolicy: always

tests/test_api.py

@@ -56,6 +56,11 @@ def client(mockmongo):
     Upload._get_collection().drop()
 
 
+def test_alive(client):
+    rv = client.get('/alive')
+    assert rv.status_code == 200
+
+
 def create_auth_headers(user):
     basic_auth_str = '%s:password' % user.email
     basic_auth_bytes = basic_auth_str.encode('utf-8')