Refactored dependency management and docker builds.

.dockerignore

@@ -2,6 +2,7 @@
 .dependencies/
 .pyenv/
 .vscode/
+.volumes/
 data/
 docs/
 infrastructure/
\ No newline at end of file

.gitignore

@@ -3,4 +3,5 @@ __pycache__
 *.pyc
 *.egg-info/
 data
-.dependencies/
\ No newline at end of file
+.dependencies/
+.volumes/
\ No newline at end of file

Dockerfile

-FROM python:3.6-stretch
-RUN rm /bin/sh && ln -s /bin/bash /bin/sh
+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
-ADD ./ /nomad
-WORKDIR /nomad
+# This dockerfile describes an image that can be used to run the nomad processing
+# worker than are capable of running all necessary celery task involving
+# upload processing, parsing, nomalization, packaging analysis files, etc.
+
+# This dockerfile was build with the following advide for small and incremental builds
+# https://blog.realkinetic.com/building-minimal-docker-containers-for-python-applications-37d0272c52f3
+
+# some dependencies with complex wheels do not build with -alpine, -slim
+FROM python:3.6-stretch as base
+
+# builder stage is used to install requirments and nomad dependencies
+# (parsers, normalizers, python-common, meta-info, etc.)
+FROM base as builder
+
+RUN mkdir /install
+WORKDIR /install
+
+COPY requirements.txt requirements.txt
+COPY requirements-worker.txt requirements-worker.txt
+COPY nomad/dependencies.py nomad/dependencies.py
+COPY nomad/config.py nomad/config.py
 
 RUN pip install -r requirements.txt
-RUN pip install -e .
 RUN pip install -r requirements-worker.txt
-RUN python nomad/parsers.py
+RUN python nomad/dependencies.py
+
+# second stage is used to install the actual code and run the celery worker as nomad user
+FROM base
+# transfer installs from builder stage
+COPY --from=builder /install /install
+COPY --from=builder /usr/local/lib/python3.6/site-packages /usr/local/lib/python3.6/site-packages
+COPY . /app
+WORKDIR /app
+
+RUN pip install -e .
 
 RUN useradd -ms /bin/bash nomad
+RUN mkdir -p /app/.volumes/fs; chown -R nomad /app/.volumes/fs
 USER nomad
+VOLUME /app/.volumes/fs

infrastructure/docker-compose.yml

@@ -24,7 +24,7 @@ services:
         ports:
             - 9007:9000
         volumes:
-            - ./data/minio:/data
+            - ../.volumes/minio:/data
             - ./config/minio:/root/.minio
         environment:
             - "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE"
@@ -49,7 +49,7 @@ services:
         image: 'redis:3.0-alpine'
         command: redis-server --appendonly yes
         volumes:
-            - './data/redis:/data'
+            - '../.volumes/redis:/data'
         ports:
             - '6379:6379'
 
@@ -73,4 +73,6 @@ services:
             - redis
             - rabbitmq
             - elk
+        volumes:
+            - '../.volumes/fs:/app/.volumes/fs'
         command: celery worker -l debug -A nomad.processing

nomad/config.py

@@ -38,7 +38,7 @@ minio = MinioConfig(
     secret='wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
 )
 fs = FSConfig(
-    tmp='./infrastructure/data/tmp'
+    tmp='.volumes/fs'
 )
 logstash = LogstashConfig(
     enabled=False,

nomad/parsers.py → nomad/dependencies.py

@@ -13,8 +13,8 @@
 # limitations under the License.
 
 """
-Integration of parsers into the processing
-==========================================
+Integration of nomad projects into the processing
+=================================================
 
 Parsers are developed as independed, individual python programs in their own GIT repositories.
 They are build on a common modules called *python-common*, also in a separate GIT.
@@ -92,14 +92,14 @@ class PythonGit():
         pipcode = 0
 
         # some weird interaction of pip and virtualenv causes a bug that does
-        # not allow to install due to a wrong PIP_REQ_TRACKER path. This
+        # not allow to install in docker due to a wrong PIP_REQ_TRACKER path. This
         # is a workarround.
         pip_req_tracker_key = 'PIP_REQ_TRACKER'
         env = dict(os.environ)
         if pip_req_tracker_key in env:
             del(env['PIP_REQ_TRACKER'])
         pipcode = subprocess.call(
-            [sys.executable, '-m', 'pip', 'install', '--no-cache-dir'] + list(args),
+            [sys.executable, '-m', 'pip', 'install'] + list(args),
             env=env)
 
         if pipcode != 0:
@@ -192,7 +192,7 @@ class VASPRunParser(Parser):
             python_git=PythonGit(
                 name='parsers/vasp',
                 git_url='https://gitlab.mpcdf.mpg.de/nomad-lab/parser-vasp.git',
-                git_commit='master'),  # COMMIT ddf8495944fbbcb62801f69b2c2c6c3d6099129d
+                git_commit='nomad-xt'),
             parser_class_name='vaspparser.VASPParser',
             main_file_re=r'^.*\.xml$',
             main_contents_re=(
@@ -212,7 +212,7 @@ others = [
     PythonGit(
         name='nomad-meta-info',
         git_url='https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-meta-info.git',
-        git_commit='1.6.0'),  # COMMIT c99a30a907e617fb39e900d3516790311c3e5c74
+        git_commit='1.6.0'),
     PythonGit(
         name='python_common',
         git_url='https://gitlab.mpcdf.mpg.de/nomad-lab/python-common.git',

nomad/processing.py

@@ -22,7 +22,7 @@ import logstash
 
 import nomad.config as config
 import nomad.files as files
-from nomad.parsers import parsers, parser_dict
+from nomad.dependencies import parsers, parser_dict
 
 
 if config.logstash.enabled:

requirements.txt

@@ -3,4 +3,3 @@ requests
 celery[redis]
 python-logstash
 gitpython
-