Commit 8b569d93 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Refactored dependency management and docker builds.

parent 5834bcf4
......@@ -2,6 +2,7 @@
.dependencies/
.pyenv/
.vscode/
.volumes/
data/
docs/
infrastructure/
\ No newline at end of file
......@@ -3,4 +3,5 @@ __pycache__
*.pyc
*.egg-info/
data
.dependencies/
\ No newline at end of file
.dependencies/
.volumes/
\ No newline at end of file
FROM python:3.6-stretch
RUN rm /bin/sh && ln -s /bin/bash /bin/sh
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
ADD ./ /nomad
WORKDIR /nomad
# This dockerfile describes an image that can be used to run the nomad processing
# worker than are capable of running all necessary celery task involving
# upload processing, parsing, nomalization, packaging analysis files, etc.
# This dockerfile was build with the following advide for small and incremental builds
# https://blog.realkinetic.com/building-minimal-docker-containers-for-python-applications-37d0272c52f3
# some dependencies with complex wheels do not build with -alpine, -slim
FROM python:3.6-stretch as base
# builder stage is used to install requirments and nomad dependencies
# (parsers, normalizers, python-common, meta-info, etc.)
FROM base as builder
RUN mkdir /install
WORKDIR /install
COPY requirements.txt requirements.txt
COPY requirements-worker.txt requirements-worker.txt
COPY nomad/dependencies.py nomad/dependencies.py
COPY nomad/config.py nomad/config.py
RUN pip install -r requirements.txt
RUN pip install -e .
RUN pip install -r requirements-worker.txt
RUN python nomad/parsers.py
RUN python nomad/dependencies.py
# second stage is used to install the actual code and run the celery worker as nomad user
FROM base
# transfer installs from builder stage
COPY --from=builder /install /install
COPY --from=builder /usr/local/lib/python3.6/site-packages /usr/local/lib/python3.6/site-packages
COPY . /app
WORKDIR /app
RUN pip install -e .
RUN useradd -ms /bin/bash nomad
RUN mkdir -p /app/.volumes/fs; chown -R nomad /app/.volumes/fs
USER nomad
VOLUME /app/.volumes/fs
......@@ -24,7 +24,7 @@ services:
ports:
- 9007:9000
volumes:
- ./data/minio:/data
- ../.volumes/minio:/data
- ./config/minio:/root/.minio
environment:
- "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE"
......@@ -49,7 +49,7 @@ services:
image: 'redis:3.0-alpine'
command: redis-server --appendonly yes
volumes:
- './data/redis:/data'
- '../.volumes/redis:/data'
ports:
- '6379:6379'
......@@ -73,4 +73,6 @@ services:
- redis
- rabbitmq
- elk
volumes:
- '../.volumes/fs:/app/.volumes/fs'
command: celery worker -l debug -A nomad.processing
......@@ -38,7 +38,7 @@ minio = MinioConfig(
secret='wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
)
fs = FSConfig(
tmp='./infrastructure/data/tmp'
tmp='.volumes/fs'
)
logstash = LogstashConfig(
enabled=False,
......
......@@ -13,8 +13,8 @@
# limitations under the License.
"""
Integration of parsers into the processing
==========================================
Integration of nomad projects into the processing
=================================================
Parsers are developed as independed, individual python programs in their own GIT repositories.
They are build on a common modules called *python-common*, also in a separate GIT.
......@@ -92,14 +92,14 @@ class PythonGit():
pipcode = 0
# some weird interaction of pip and virtualenv causes a bug that does
# not allow to install due to a wrong PIP_REQ_TRACKER path. This
# not allow to install in docker due to a wrong PIP_REQ_TRACKER path. This
# is a workarround.
pip_req_tracker_key = 'PIP_REQ_TRACKER'
env = dict(os.environ)
if pip_req_tracker_key in env:
del(env['PIP_REQ_TRACKER'])
pipcode = subprocess.call(
[sys.executable, '-m', 'pip', 'install', '--no-cache-dir'] + list(args),
[sys.executable, '-m', 'pip', 'install'] + list(args),
env=env)
if pipcode != 0:
......@@ -192,7 +192,7 @@ class VASPRunParser(Parser):
python_git=PythonGit(
name='parsers/vasp',
git_url='https://gitlab.mpcdf.mpg.de/nomad-lab/parser-vasp.git',
git_commit='master'), # COMMIT ddf8495944fbbcb62801f69b2c2c6c3d6099129d
git_commit='nomad-xt'),
parser_class_name='vaspparser.VASPParser',
main_file_re=r'^.*\.xml$',
main_contents_re=(
......@@ -212,7 +212,7 @@ others = [
PythonGit(
name='nomad-meta-info',
git_url='https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-meta-info.git',
git_commit='1.6.0'), # COMMIT c99a30a907e617fb39e900d3516790311c3e5c74
git_commit='1.6.0'),
PythonGit(
name='python_common',
git_url='https://gitlab.mpcdf.mpg.de/nomad-lab/python-common.git',
......
......@@ -22,7 +22,7 @@ import logstash
import nomad.config as config
import nomad.files as files
from nomad.parsers import parsers, parser_dict
from nomad.dependencies import parsers, parser_dict
if config.logstash.enabled:
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment