diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 31e66b88aac611148ae5044c6fceb28b97a87ba1..cede45cc8b43ea01a2f0dcfd8e060e5cfa4e6477 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -13,6 +13,7 @@ stages:
- qa
- test
- release
+ - deploy
variables:
TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:test
@@ -81,4 +82,17 @@ release:
- docker pull $FRONTEND_TEST_IMAGE
- docker tag $FRONTEND_TEST_IMAGE $FRONTEND_RELEASE_IMAGE
- docker push $FRONTEND_RELEASE_IMAGE
+ only: master
+
+deploy:
+ stage: deploy
+ script:
+ - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
+ - cd infrastructure/nomad
+ # secret .env file is stored on the "production" machine and is bind mounted into the
+ # runner container at /nomad/config/.env
+ - cp /nomad/config/.env .
+ - docker-compose -f docker-compose.yml -f docker-compose.prod.yml build
+ - docker-compose -f docker-compose.yml -f docker-compose.prod.yml up --no-deps -d api worker gui
+ only: master
when: manual
diff --git a/infrastructure/nomad/.env_development b/infrastructure/nomad/.env_development
new file mode 100644
index 0000000000000000000000000000000000000000..22355b43a066e363ef95ee0cdb68f450d4843aca
--- /dev/null
+++ b/infrastructure/nomad/.env_development
@@ -0,0 +1,16 @@
+# copy this to .env and modify
+
+# in production the values must be replaced with real host, cookies, and secrets
+# do not check this information into git
+
+VOLUME_BINDS=../../.volumes
+EXTERNAL_HOST=localhost
+EXTERNAL_PORT=80
+
+RABBITMQ_ERLANG_COOKIE=SWQOKODSQALRPCLNMEQG
+RABBITMQ_DEFAULT_USER=rabbitmq
+RABBITMQ_DEFAULT_PASS=rabbitmq
+RABBITMQ_DEFAULT_VHOST=/
+
+API_SECRET=defaultApiSecret
+
diff --git a/infrastructure/nomad/.env_local b/infrastructure/nomad/.env_local
deleted file mode 100644
index 1ce26a269edcc5c6193ef89185e1b67c234e23ca..0000000000000000000000000000000000000000
--- a/infrastructure/nomad/.env_local
+++ /dev/null
@@ -1,13 +0,0 @@
-RABBITMQ_HOST_PORT=5672
-REDIS_HOST_PORT=6379
-ELASTIC_HOST_PORT=9200
-MONGO_HOST_PORT=27017
-KIBANA_HOST_PORT=5601
-
-API_HOST_PORT=8000
-GUI_HOST_PORT=8080
-APP_HOST_PORT=80
-
-VOLUME_BINDS=../../.volumes
-EXTERNAL_HOST=localhost
-EXTERNAL_PORT=80
\ No newline at end of file
diff --git a/infrastructure/nomad/.env_processing b/infrastructure/nomad/.env_processing
deleted file mode 100644
index d9d6b3e2828484e73d08a0bdc77042a41fc98c5a..0000000000000000000000000000000000000000
--- a/infrastructure/nomad/.env_processing
+++ /dev/null
@@ -1,13 +0,0 @@
-RABBITMQ_HOST_PORT=15672
-REDIS_HOST_PORT=16379
-ELASTIC_HOST_PORT=19200
-MONGO_HOST_PORT=37017
-KIBANA_HOST_PORT=15601
-
-API_HOST_PORT=18000
-GUI_HOST_PORT=18080
-APP_HOST_PORT=0.0.0.0:10080
-
-VOLUME_BINDS=/scratch/nomad_FAIR
-EXTERNAL_HOST=enc-staging-nomad.esc.rzg.mpg.de
-EXTERNAL_PORT=80
diff --git a/infrastructure/nomad/docker-compose.dev.yml b/infrastructure/nomad/docker-compose.dev.yml
new file mode 100644
index 0000000000000000000000000000000000000000..32b59b9678cd18a6122d6bae0362d12bda15a2fa
--- /dev/null
+++ b/infrastructure/nomad/docker-compose.dev.yml
@@ -0,0 +1,54 @@
+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: '3.4'
+
+services:
+ # broker for celery
+ rabbitmq:
+ ports:
+ - 5672:5672
+
+ # the search engine
+ elastic:
+ ports:
+ - 9200:9200
+
+ # the user data db
+ mongo:
+ ports:
+ - 27017:27017
+
+ # nomad processing worker
+ worker:
+ restart: 'no'
+ build: ../../
+ image: nomad/backend
+
+ # nomad api
+ api:
+ restart: 'no'
+ image: nomad/backend
+ depends_on:
+ - worker
+ ports:
+ - 8000:8000
+
+ # nomad gui
+ gui:
+ restart: 'no'
+ build: ../../gui
+ image: nomad/frontend
+ ports:
+ - 8080:8000
diff --git a/infrastructure/nomad/docker-compose.prod.yml b/infrastructure/nomad/docker-compose.prod.yml
new file mode 100644
index 0000000000000000000000000000000000000000..9c1442fefcc14a2064a38d8ab224009b5925da75
--- /dev/null
+++ b/infrastructure/nomad/docker-compose.prod.yml
@@ -0,0 +1,48 @@
+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: '3.4'
+
+services:
+ # the search engine
+ elastic:
+ ports:
+ - 19200:9200
+
+ # the user data db
+ mongo:
+ ports:
+ - 37017:27017
+
+ # used for centralized logging
+ elk:
+ restart: always
+ build: ../elk/
+ image: nomad/elk
+ container_name: nomad_elk
+ expose:
+ - 5000 # logstash beats
+ - 5044 # logstash tcp
+ volumes:
+ - nomad_elk:/var/lib/elasticsearch
+ ports:
+ - 15601:5601 # kibana web
+
+ api:
+ links:
+ - elk
+
+ worker:
+ links:
+ - elk
diff --git a/infrastructure/nomad/docker-compose.yml b/infrastructure/nomad/docker-compose.yml
index 20f0d6d48d384fe15a28d9cfbfa7203372b38add..d614dba12bb10e76242c0b63f2956c253898a18e 100644
--- a/infrastructure/nomad/docker-compose.yml
+++ b/infrastructure/nomad/docker-compose.yml
@@ -27,12 +27,10 @@ services:
image: rabbitmq
container_name: nomad_rabbitmq
environment:
- - RABBITMQ_ERLANG_COOKIE=SWQOKODSQALRPCLNMEQG
- - RABBITMQ_DEFAULT_USER=rabbitmq
- - RABBITMQ_DEFAULT_PASS=rabbitmq
+ - RABBITMQ_ERLANG_COOKIE=${RABBITMQ_ERLANG_COOKIE}
+ - RABBITMQ_DEFAULT_USER=${RABBITMQ_DEFAULT_USER}
+ - RABBITMQ_DEFAULT_PASS=${RABBITMQ_DEFAULT_PASS}
- RABBITMQ_DEFAULT_VHOST=/
- ports:
- - ${RABBITMQ_HOST_PORT}:5672
volumes:
- nomad_rabbitmq:/var/lib/rabbitmq
@@ -42,8 +40,6 @@ services:
container_name: nomad_elastic
volumes:
- nomad_elastic:/usr/share/elasticsearch/data
- ports:
- - ${ELASTIC_HOST_PORT}:9200
# the user data db
mongo:
@@ -54,29 +50,12 @@ services:
- MONGO_LOG_DIR=/dev/null
volumes:
- nomad_mongo:/data/db
- ports:
- - ${MONGO_HOST_PORT}:27017
command: mongod --smallfiles --logpath=/dev/null # --quiet
- # used for centralized logging
- elk:
- restart: always
- build: ../elk/
- image: nomad/elk
- container_name: nomad_elk
- expose:
- - 5000 # logstash beats
- - 5044 # logstash tcp
- volumes:
- - nomad_elk:/var/lib/elasticsearch
- ports:
- - ${KIBANA_HOST_PORT}:5601 # kibana web
-
# nomad processing worker
worker:
restart: always
- build: ../../
- image: nomad/backend
+ image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest
container_name: nomad_worker
environment:
<<: *nomad_backend_env
@@ -85,7 +64,6 @@ services:
- rabbitmq
- elastic
- mongo
- - elk
volumes:
- ${VOLUME_BINDS}/fs:/app/.volumes/fs
command: python -m celery worker -l info -A nomad.processing
@@ -93,9 +71,7 @@ services:
# nomad api
api:
restart: always
- image: nomad/backend
- depends_on:
- - worker
+ image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest
container_name: nomad_api
environment:
<<: *nomad_backend_env
@@ -103,24 +79,20 @@ services:
NOMAD_API_HOST: ${EXTERNAL_HOST}
NOMAD_API_PORT: ${EXTERNAL_PORT}
NOMAD_SERVICE: nomad_api
+ NOMAD_API_SECRET: ${API_SECRET}
links:
- rabbitmq
- elastic
- mongo
- - elk
- ports:
- - ${API_HOST_PORT}:8000
volumes:
- ${VOLUME_BINDS}/fs:/app/.volumes/fs
command: python -m gunicorn.app.wsgiapp -w 4 -b 0.0.0.0:8000 nomad.api:app
# nomad gui
gui:
- build: ../../gui
- image: nomad/frontend
+ restart: always
+ image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:latest
container_name: nomad_gui
- ports:
- - ${GUI_HOST_PORT}:8080
volumes:
- ./gui.conf:/etc/nginx/conf.d/default.conf:ro
command: nginx -g 'daemon off;'
@@ -131,11 +103,10 @@ services:
image: nginx:1.13.9-alpine
container_name: nomad_proxy
links:
- - elk
- gui
- api
ports:
- - ${APP_HOST_PORT}:80
+ - 80:80
volumes:
- ./nginx.conf:/etc/nginx/conf.d/default.conf:ro
command: nginx -g 'daemon off;'
@@ -146,5 +117,3 @@ volumes:
nomad_redis:
nomad_rabbitmq:
nomad_elk:
-
-
diff --git a/nomad/config.py b/nomad/config.py
index 7d8f5f3bc6eb391b2d5c799c1e8e40f2ce492a9d..742d984d068803d378eb3639edbf43dfe53191c7 100644
--- a/nomad/config.py
+++ b/nomad/config.py
@@ -83,5 +83,5 @@ services = NomadServicesConfig(
api_host=os.environ.get('NOMAD_API_HOST', 'localhost'),
api_port=int(os.environ.get('NOMAD_API_PORT', 8000)),
api_base_path=os.environ.get('NOMAD_API_BASE_PATH', '/nomad/api'),
- api_secret='the quick fox jumps over something'
+ api_secret=os.environ.get('NOMAD_API_SECRET', 'defaultApiSecret')
)
diff --git a/tests/processing/test_base.py b/tests/processing/test_base.py
index 44d44aa3773754beceb7bf846af1c145cdf75c9b..b777d3d658aa7a0d1313b3650ee0bc6a25e46a72 100644
--- a/tests/processing/test_base.py
+++ b/tests/processing/test_base.py
@@ -140,5 +140,6 @@ def test_counter(worker, no_warn):
p.spawn_children()
p.block_until_complete()
+ p = ParentProc.get(p.id)
assert_proc(p, 'join')
assert p.joined