Refactored infrastructure to support deployment.

.gitlab-ci.yml

@@ -13,6 +13,7 @@ stages:
   - qa
   - test
   - release
+  - deploy
 
 variables:
   TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:test
@@ -81,4 +82,17 @@ release:
     - docker pull $FRONTEND_TEST_IMAGE
     - docker tag $FRONTEND_TEST_IMAGE $FRONTEND_RELEASE_IMAGE
     - docker push $FRONTEND_RELEASE_IMAGE
+  only: master
+
+deploy:
+  stage: deploy
+  script:
+    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
+    - cd infrastructure/nomad
+    # secret .env file is stored on the "production" machine and is bind mounted into the
+    # runner container at /nomad/config/.env
+    - cp /nomad/config/.env .
+    - docker-compose -f docker-compose.yml -f docker-compose.prod.yml build
+    - docker-compose -f docker-compose.yml -f docker-compose.prod.yml up --no-deps -d api worker gui
+  only: master
   when: manual

infrastructure/nomad/.env_development

+# copy this to .env and modify
+
+# in production the values must be replaced with real host, cookies, and secrets
+# do not check this information into git
+
+VOLUME_BINDS=../../.volumes
+EXTERNAL_HOST=localhost
+EXTERNAL_PORT=80
+
+RABBITMQ_ERLANG_COOKIE=SWQOKODSQALRPCLNMEQG
+RABBITMQ_DEFAULT_USER=rabbitmq
+RABBITMQ_DEFAULT_PASS=rabbitmq
+RABBITMQ_DEFAULT_VHOST=/
+
+API_SECRET=defaultApiSecret
+

infrastructure/nomad/.env_local

-RABBITMQ_HOST_PORT=5672
-REDIS_HOST_PORT=6379
-ELASTIC_HOST_PORT=9200
-MONGO_HOST_PORT=27017
-KIBANA_HOST_PORT=5601
-
-API_HOST_PORT=8000
-GUI_HOST_PORT=8080
-APP_HOST_PORT=80
-
-VOLUME_BINDS=../../.volumes
-EXTERNAL_HOST=localhost
-EXTERNAL_PORT=80
\ No newline at end of file

infrastructure/nomad/.env_processing

-RABBITMQ_HOST_PORT=15672
-REDIS_HOST_PORT=16379
-ELASTIC_HOST_PORT=19200
-MONGO_HOST_PORT=37017
-KIBANA_HOST_PORT=15601
-
-API_HOST_PORT=18000
-GUI_HOST_PORT=18080
-APP_HOST_PORT=0.0.0.0:10080
-
-VOLUME_BINDS=/scratch/nomad_FAIR
-EXTERNAL_HOST=enc-staging-nomad.esc.rzg.mpg.de
-EXTERNAL_PORT=80

infrastructure/nomad/docker-compose.dev.yml

+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: '3.4'
+
+services:
+    # broker for celery
+    rabbitmq:
+        ports:
+            - 5672:5672
+
+    # the search engine
+    elastic:
+        ports:
+            - 9200:9200
+
+    # the user data db
+    mongo:
+        ports:
+            - 27017:27017
+
+    # nomad processing worker
+    worker:
+        restart: 'no'
+        build: ../../
+        image: nomad/backend
+
+    # nomad api
+    api:
+        restart: 'no'
+        image: nomad/backend
+        depends_on:
+            - worker
+        ports:
+            - 8000:8000
+
+    # nomad gui
+    gui:
+        restart: 'no'
+        build: ../../gui
+        image: nomad/frontend
+        ports:
+            - 8080:8000

infrastructure/nomad/docker-compose.prod.yml

+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: '3.4'
+
+services:
+    # the search engine
+    elastic:
+        ports:
+            - 19200:9200
+
+    # the user data db
+    mongo:
+        ports:
+            - 37017:27017
+
+    # used for centralized logging
+    elk:
+        restart: always
+        build: ../elk/
+        image: nomad/elk
+        container_name: nomad_elk
+        expose:
+            - 5000 # logstash beats
+            - 5044 # logstash tcp
+        volumes:
+            - nomad_elk:/var/lib/elasticsearch
+        ports:
+            - 15601:5601  # kibana web
+
+    api:
+        links:
+            - elk
+
+    worker:
+        links:
+            - elk

infrastructure/nomad/docker-compose.yml

@@ -27,12 +27,10 @@ services:
         image: rabbitmq
         container_name: nomad_rabbitmq
         environment:
-            - RABBITMQ_ERLANG_COOKIE=SWQOKODSQALRPCLNMEQG
-            - RABBITMQ_DEFAULT_USER=rabbitmq
-            - RABBITMQ_DEFAULT_PASS=rabbitmq
+            - RABBITMQ_ERLANG_COOKIE=${RABBITMQ_ERLANG_COOKIE}
+            - RABBITMQ_DEFAULT_USER=${RABBITMQ_DEFAULT_USER}
+            - RABBITMQ_DEFAULT_PASS=${RABBITMQ_DEFAULT_PASS}
             - RABBITMQ_DEFAULT_VHOST=/
-        ports:
-            - ${RABBITMQ_HOST_PORT}:5672
         volumes:
             - nomad_rabbitmq:/var/lib/rabbitmq
 
@@ -42,8 +40,6 @@ services:
         container_name: nomad_elastic
         volumes:
             - nomad_elastic:/usr/share/elasticsearch/data
-        ports:
-            - ${ELASTIC_HOST_PORT}:9200
 
     # the user data db
     mongo:
@@ -54,29 +50,12 @@ services:
             - MONGO_LOG_DIR=/dev/null
         volumes:
             - nomad_mongo:/data/db
-        ports:
-            - ${MONGO_HOST_PORT}:27017
         command: mongod --smallfiles --logpath=/dev/null # --quiet
 
-    # used for centralized logging
-    elk:
-        restart: always
-        build: ../elk/
-        image: nomad/elk
-        container_name: nomad_elk
-        expose:
-            - 5000 # logstash beats
-            - 5044 # logstash tcp
-        volumes:
-            - nomad_elk:/var/lib/elasticsearch
-        ports:
-            - ${KIBANA_HOST_PORT}:5601  # kibana web
-
     # nomad processing worker
     worker:
         restart: always
-        build: ../../
-        image: nomad/backend
+        image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest
         container_name: nomad_worker
         environment:
             <<: *nomad_backend_env
@@ -85,7 +64,6 @@ services:
             - rabbitmq
             - elastic
             - mongo
-            - elk
         volumes:
             - ${VOLUME_BINDS}/fs:/app/.volumes/fs
         command: python -m celery worker -l info -A nomad.processing
@@ -93,9 +71,7 @@ services:
     # nomad api
     api:
         restart: always
-        image: nomad/backend
-        depends_on:
-            - worker
+        image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest
         container_name: nomad_api
         environment:
             <<: *nomad_backend_env
@@ -103,24 +79,20 @@ services:
             NOMAD_API_HOST: ${EXTERNAL_HOST}
             NOMAD_API_PORT: ${EXTERNAL_PORT}
             NOMAD_SERVICE: nomad_api
+            NOMAD_API_SECRET: ${API_SECRET}
         links:
             - rabbitmq
             - elastic
             - mongo
-            - elk
-        ports:
-            - ${API_HOST_PORT}:8000
         volumes:
             - ${VOLUME_BINDS}/fs:/app/.volumes/fs
         command: python -m gunicorn.app.wsgiapp -w 4 -b 0.0.0.0:8000 nomad.api:app
 
     # nomad gui
     gui:
-        build: ../../gui
-        image: nomad/frontend
+        restart: always
+        image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:latest
         container_name: nomad_gui
-        ports:
-            - ${GUI_HOST_PORT}:8080
         volumes:
             - ./gui.conf:/etc/nginx/conf.d/default.conf:ro
         command: nginx -g 'daemon off;'
@@ -131,11 +103,10 @@ services:
         image: nginx:1.13.9-alpine
         container_name: nomad_proxy
         links:
-            - elk
             - gui
             - api
         ports:
-            - ${APP_HOST_PORT}:80
+            - 80:80
         volumes:
             - ./nginx.conf:/etc/nginx/conf.d/default.conf:ro
         command: nginx -g 'daemon off;'
@@ -146,5 +117,3 @@ volumes:
     nomad_redis:
     nomad_rabbitmq:
     nomad_elk:
-
-

nomad/config.py

@@ -83,5 +83,5 @@ services = NomadServicesConfig(
     api_host=os.environ.get('NOMAD_API_HOST', 'localhost'),
     api_port=int(os.environ.get('NOMAD_API_PORT', 8000)),
     api_base_path=os.environ.get('NOMAD_API_BASE_PATH', '/nomad/api'),
-    api_secret='the quick fox jumps over something'
+    api_secret=os.environ.get('NOMAD_API_SECRET', 'defaultApiSecret')
 )

tests/processing/test_base.py

@@ -140,5 +140,6 @@ def test_counter(worker, no_warn):
     p.spawn_children()
     p.block_until_complete()
 
+    p = ParentProc.get(p.id)
     assert_proc(p, 'join')
     assert p.joined