Commit 34436ba8 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Some docker/doc refactoring, renaming of nomadxt -> nomad(-FAIR).

parent 689445ae
......@@ -16,19 +16,19 @@
# - nomad processing worker
# - nomad upload handler that initiates processing after upload
# - nomad api
#
# The dockerfile is multistages to getaway with building on a larger base image.
# using the base image with most requirements already installed
FROM nomad_requirements:latest as requirements
# we use slim for the final image
FROM python:3.6-slim as final
# The dockerfile is multistaged to help docker with caching unnecessary steps
# creating a base image with most requirements already installed
FROM python:3.6-stretch as requirements
RUN mkdir /install
WORKDIR /install
COPY requirements.txt requirements.txt
COPY requirements-dep.txt requirements-dep.txt
RUN pip install -r requirements.txt
RUN pip install -r requirements-dep.txt
# dependency stage is used to install nomad coe projects
FROM requirements as dependencies
# do stuff
WORKDIR /install
COPY nomad/dependencies.py nomad/dependencies.py
COPY nomad/config.py nomad/config.py
......@@ -36,6 +36,9 @@ COPY requirements.txt requirements.txt
RUN pip install -r requirements.txt
RUN python nomad/dependencies.py
# we use slim for the final image
FROM python:3.6-slim as final
# last stage is used to install the actual code, nomad user, volumes
FROM final
# transfer installed packages from dependency stage
......
......@@ -95,26 +95,17 @@ you change things, debug, and re-run things quickly. The later one brings you
closer to the environement that will be used to run nomad in production.
### Docker images for nomad
There are currently three different images and respectively three different docker files:
`requirements.Dockerfile`, `backend.Dockerfile`, and `frontend.Dockerfile`.
There are currently two different images and respectively two different docker files:
`backend.Dockerfile`, and `frontend.Dockerfile`.
Nomad comprises currently three services, the *handler* (deals with user uploads),
the *worker* (does the actual processing), and the *api*. Those services can be
run from one image that have the nomad python code and all dependencies installed. This
is covered by the `backend.Dockerfile`.
The `requirements.Dockerfile` builds an image that has all dependencies pre installed.
We keep it separate, because the dependencies change rather seldomly and we do not want to
reinstall them all the time, we need to build new images.
The fontend image is only for building and serving the gui.
Build the requirements image tagged `nomad_requirements`:
```
docker build -t nomad_requirements -f requirements.Dockerfile .
```
The other images are build via *docker-compose* and don't have to be created manually.
The images are build via *docker-compose* and don't have to be created manually.
### Build with docker-compose
......@@ -162,6 +153,14 @@ docker-compose up worker handler
docker-compose up api gui proxy
```
### Configure the containers
The *docker-compose* takes some configuration from the environment. Environment
variables are set in `.env`, which is a link to either `.env_local` (intended to run
nomad for development on a local computer) and `.env_processing` (indented to run
nomad on nomad in *'prodcution'*, currently on the enc pre-processing machine).
You can configure host ports, volume locations for host bindings, and these sort of things.
## Accessing 3'rd party services
Usually these services only used by the nomad containers, but sometimes you also
......
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This docker image is used for the GUI container that serves the static GUI
# files.
#
# This is a multistage build, one stage for building the GUI and one final stage
# intended for the actual GUI container that serves the GUI.
# build environment
FROM node:latest as builder
RUN mkdir -p /nomad/app
......
......@@ -12,7 +12,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
version: '3'
version: '3.4'
x-common-variables: &nomad_backend_env
NOMAD_MINIO_PORT: 9000
NOMAD_MINIO_HOST: minio
NOMAD_RABBITMQ_HOST: rabbitmq
NOMAD_LOGSTASH_HOST: elk
NOMAD_ELASTIC_HOST: elastic
NOMAD_MONGO_HOST: mongo
services:
# an object store for uploads
......@@ -85,6 +93,7 @@ services:
elk:
restart: always
build: ../elk/
image: nomad/elk
container_name: nomad_elk
expose:
- 5000 # logstash beats
......@@ -100,15 +109,11 @@ services:
build:
context: ../../
dockerfile: backend.Dockerfile
image: nomad/backend
container_name: nomad_worker
environment:
- NOMAD_MINIO_PORT=9000
- NOMAD_MINIO_HOST=minio
- NOMAD_RABBITMQ_HOST=rabbitmq
- NOMAD_LOGSTASH_HOST=elk
- NOMAD_ELASTIC_HOST=elastic
- NOMAD_MONGO_HOST=mongo
- NOMAD_SERVICE=nomad_worker
<<: *nomad_backend_env
NOMAD_SERVICE: nomad_worker
links:
- minio
- rabbitmq
......@@ -122,18 +127,13 @@ services:
# nomad upload handler
handler:
restart: always
build:
context: ../../
dockerfile: backend.Dockerfile
image: nomad/backend
depends_on:
- worker
container_name: nomad_handler
environment:
- NOMAD_MINIO_PORT=9000
- NOMAD_MINIO_HOST=minio
- NOMAD_RABBITMQ_HOST=rabbitmq
- NOMAD_LOGSTASH_HOST=elk
- NOMAD_MONGO_HOST=mongo
- NOMAD_ELASTIC_HOST=elastic
- NOMAD_SERVICE=nomad_handler
<<: *nomad_backend_env
NOMAD_SERVICE: nomad_handler
links:
- minio
- rabbitmq
......@@ -145,22 +145,17 @@ services:
# nomad api
api:
restart: always
build:
context: ../../
dockerfile: backend.Dockerfile
image: nomad/backend
depends_on:
- worker
container_name: nomad_api
environment:
- NOMAD_MINIO_PORT=9000
- NOMAD_MINIO_HOST=minio
- NOMAD_RABBITMQ_HOST=rabbitmq
- NOMAD_LOGSTASH_HOST=elk
- NOMAD_ELASTIC_HOST=elastic
- NOMAD_MONGO_HOST=mongo
- NOMAD_API_BASE_PATH=/nomad/api
- NOMAD_OBJECTS_HOST=${EXTERNAL_HOST}
- NOMAD_OBJECTS_PORT=${EXTERNAL_PORT}
- NOMAD_OBJECTS_BASE_PATH=/nomad/objects
- NOMAD_SERVICE=nomad_api
<<: *nomad_backend_env
NOMAD_API_BASE_PATH: /nomad/api
NOMAD_OBJECTS_HOST: ${EXTERNAL_HOST}
NOMAD_OBJECTS_PORT: ${EXTERNAL_PORT}
NOMAD_OBJECTS_BASE_PATH: /nomad/objects
NOMAD_SERVICE: nomad_api
links:
- minio
- rabbitmq
......@@ -176,6 +171,7 @@ services:
build:
context: ../../
dockerfile: frontend.Dockerfile
image: nomad/frontend
container_name: nomad_gui
ports:
- ${GUI_HOST_PORT}:8080
......
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This dockerfile creates a python image with most rarely changing requirements
# preinstalled, to speed up the build of the actual nomad-xt image.
# some dependencies with complex wheels do not build with -alpine, -slim
FROM python:3.6-stretch as builder
# do stuff
RUN mkdir /install
WORKDIR /install
COPY requirements.txt requirements.txt
COPY requirements-dep.txt requirements-dep.txt
RUN pip install -r requirements.txt
RUN pip install -r requirements-dep.txt
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment