From 1ff41c24005ce2558fa726c5347b050e297ec73c Mon Sep 17 00:00:00 2001
From: Markus Scheidgen <markus.scheidgen@gmail.com>
Date: Fri, 21 Sep 2018 10:58:30 +0200
Subject: [PATCH] Refactored docker images.
---
.dockerignore | 9 +++-
backend.Dockerfile => Dockerfile | 59 +++++++++++++--------------
docs/conf.py | 17 +++++---
frontend.Dockerfile => gui/Dockerfile | 11 +++--
gui/gitinfo.sh | 2 +
gui/package.json | 4 +-
nomad/dependencies.py | 23 ++++++++---
7 files changed, 74 insertions(+), 51 deletions(-)
rename backend.Dockerfile => Dockerfile (53%)
rename frontend.Dockerfile => gui/Dockerfile (83%)
create mode 100644 gui/gitinfo.sh
diff --git a/.dockerignore b/.dockerignore
index 3118add92b..8a3681360e 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -2,10 +2,15 @@
.pyenv/
.vscode/
.volumes/
+.git/
+.mypy_cache/
data/
docs/.build
-gui/node_modules/
-gui/build/
+docs/.static
+docs/*.graffle
+gui/
infrastructure/
__pycache__/
*.pyc
+NOMAD.egg-info/
+.coverage
diff --git a/backend.Dockerfile b/Dockerfile
similarity index 53%
rename from backend.Dockerfile
rename to Dockerfile
index 08b6f602f8..17639ce762 100644
--- a/backend.Dockerfile
+++ b/Dockerfile
@@ -17,49 +17,48 @@
# - nomad upload handler that initiates processing after upload
# - nomad api
-# The dockerfile is multistaged to help docker with caching unnecessary steps
-# creating a base image with most requirements already installed
-FROM python:3.6-stretch as requirements
+# The dockerfile is multistaged to use a fat, more convinient build image and
+# copy only necessities to a slim final image
+
+# We use slim for the final image
+FROM python:3.6-slim as final
+
+# First, build everything in a build image
+FROM python:3.6-stretch as build
+# Make will be necessary to build the docs with sphynx
+RUN apt-get update && apt-get install -y make
RUN mkdir /install
WORKDIR /install
-COPY requirements.txt requirements.txt
+# We also install the -dev dependencies, to use this image for test and qa
+COPY requirements-dev.txt requirements-dev.txt
+RUN pip install -r requirements-dev.txt
COPY requirements-dep.txt requirements-dep.txt
-RUN pip install -r requirements.txt
RUN pip install -r requirements-dep.txt
-
-# dependency stage is used to install nomad coe projects
-FROM requirements as dependencies
-WORKDIR /install
-COPY nomad/dependencies.py nomad/dependencies.py
-COPY nomad/config.py nomad/config.py
COPY requirements.txt requirements.txt
RUN pip install -r requirements.txt
+# Use docker build --build-args CACHEBUST=2 to not cache this (e.g. when you know deps have changed)
+ARG CACHEBUST=1
+COPY nomad/dependencies.py /install/nomad/dependencies.py
+COPY nomad/config.py /install/nomad/config.py
RUN python nomad/dependencies.py
+# do that after the dependencies to use docker's layer caching
+COPY . /install
+RUN pip install .
+WORKDIR /install/docs
+RUN make html
-# we use slim for the final image
-FROM python:3.6-slim as final
-
-# last stage is used to install the actual code, nomad user, volumes
+# Second, create a slim final image
FROM final
-# transfer installed packages from dependency stage
-COPY --from=dependencies /usr/local/lib/python3.6/site-packages /usr/local/lib/python3.6/site-packages
-COPY --from=dependencies /usr/local/bin/sphinx-build /usr/local/bin/sphinx-build
-# we also need to copy the install dir, since nomad coe deps are installed with -e
-# TODO that should be changed in production!
-COPY --from=dependencies /install /install
-
-# do stuff
-RUN apt-get update && apt-get install -y make
+# copy the sources for tests, coverage, qa, etc.
COPY . /app
WORKDIR /app
-
+# transfer installed packages from dependency stage
+COPY --from=build /usr/local/lib/python3.6/site-packages /usr/local/lib/python3.6/site-packages
# copy the meta-info, since it files are loaded via relative paths. TODO that should change.
-COPY --from=dependencies /install/.dependencies/nomad-meta-info /app/.dependencies/nomad-meta-info
+COPY --from=build /install/.dependencies/nomad-meta-info /app/.dependencies/nomad-meta-info
+# copy the documentation, its files will be served by the API
+COPY --from=build /install/docs/.build /app/docs/.build
-RUN pip install -e .
-WORKDIR /app/docs
-RUN make html
-WORKDIR /app
RUN useradd -ms /bin/bash nomad
RUN mkdir -p /app/.volumes/fs; chown -R nomad /app/.volumes/fs
USER nomad
diff --git a/docs/conf.py b/docs/conf.py
index c158fabb3e..b487817678 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -15,12 +15,17 @@ import os
import sys
from recommonmark.transform import AutoStructify
-sys.path.insert(0, os.path.abspath('../nomad'))
+sys.path.insert(0, os.path.abspath('..'))
+# TODO, once the normalizers are self contained in their own gits, this should not be
+# necessary anymore
+sys.path.insert(0, os.path.abspath('../.dependencies/normalizers/stats/normalizer/normalizer-stats'))
+sys.path.insert(0, os.path.abspath('../.dependencies/normalizers/symmetry/normalizer/normalizer-symmetry'))
+sys.path.insert(0, os.path.abspath('../.dependencies/normalizers/system-type/normalizer/normalizer-system-type'))
# -- Project information -----------------------------------------------------
-project = 'NOMAD-XT'
+project = 'nomad-FAIR'
copyright = '2018, the NOMAD developers'
author = 'the NOMAD developers'
@@ -141,7 +146,7 @@ latex_elements = {
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
- (master_doc, 'NOMAD-XT.tex', 'NOMAD-XT Documentation',
+ (master_doc, 'nomad-FAIR.tex', 'nomad-FAIR Documentation',
'the NOMAD developers', 'manual'),
]
@@ -151,7 +156,7 @@ latex_documents = [
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
- (master_doc, 'nomad', 'NOMAD-XT Documentation',
+ (master_doc, 'nomad', 'nomad-FAIR Documentation',
[author], 1)
]
@@ -162,8 +167,8 @@ man_pages = [
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
- (master_doc, 'NOMAD-XT', 'NOMAD-XT Documentation',
- author, 'NOMAD-XT', 'One line description of project.',
+ (master_doc, 'nomad-FAIR', 'nomad-FAIR Documentation',
+ author, 'nomad-FAIR', 'One line description of project.',
'Miscellaneous'),
]
diff --git a/frontend.Dockerfile b/gui/Dockerfile
similarity index 83%
rename from frontend.Dockerfile
rename to gui/Dockerfile
index dddaab8d69..e13e34d145 100644
--- a/frontend.Dockerfile
+++ b/gui/Dockerfile
@@ -19,19 +19,18 @@
# intended for the actual GUI container that serves the GUI.
# build environment
-FROM node:latest as builder
+FROM node:latest as build
RUN mkdir -p /nomad/app
WORKDIR /nomad/app
ENV PATH /nomad/app/node_modules/.bin:$PATH
-COPY gui/package.json /nomad/app/package.json
-COPY gui/yarn.lock /nomad/app/yarn.lock
+COPY package.json /nomad/app/package.json
+COPY yarn.lock /nomad/app/yarn.lock
RUN yarn
-COPY gui /nomad/app
-COPY .git /nomad
+COPY . /nomad/app
RUN yarn build
# production environment
FROM nginx:1.13.9-alpine
-COPY --from=builder /nomad/app/build /app/nomad
+COPY --from=build /nomad/app/build /app/nomad
CMD ["nginx", "-g", "daemon off;"]
diff --git a/gui/gitinfo.sh b/gui/gitinfo.sh
new file mode 100644
index 0000000000..a4755c6c87
--- /dev/null
+++ b/gui/gitinfo.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+echo \"{ \\\"log\\\": \\\"$(git log -1 --oneline)\\\", \\\"ref\\\": \\\"$(git describe --all)\\\", \\\"version\\\": \\\"$(git describe)\\\" }\" > src/gitinfo.json
\ No newline at end of file
diff --git a/gui/package.json b/gui/package.json
index 8ce96ad07c..ef0c4b41dd 100644
--- a/gui/package.json
+++ b/gui/package.json
@@ -25,8 +25,8 @@
"scripts": {
"metainfo": "git clone --single-branch -b nomad-xt http://gitlab.mpcdf.mpg.de/nomad-lab/nomad-meta-info.git --depth=1 public/metainfo",
"gitinfo": "echo \"{ \\\"log\\\": \\\"$(git log -1 --oneline)\\\", \\\"ref\\\": \\\"$(git describe --all)\\\", \\\"version\\\": \\\"$(git describe)\\\" }\" > src/gitinfo.json",
- "start": "yarn metainfo; yarn gitinfo; react-scripts start",
- "build": "yarn metainfo; yarn gitinfo; react-scripts build",
+ "start": "yarn metainfo; react-scripts start",
+ "build": "yarn metainfo; react-scripts build",
"test": "react-scripts test --env=jsdom",
"eject": "react-scripts eject"
},
diff --git a/nomad/dependencies.py b/nomad/dependencies.py
index d684f4c446..2288728f44 100644
--- a/nomad/dependencies.py
+++ b/nomad/dependencies.py
@@ -95,10 +95,13 @@ class PythonGit():
raise PythonGitError(
'Could not install (pip return code=%s)' % pipcode, repo=self)
- def prepare(self) -> None:
+ def prepare(self, dev: bool = False) -> None:
"""
Makes sure that the repository is fetched, at the right commit, and installed.
+ Arguments:
+ dev (bool): Indicate dev install (uses pip with -e). Default is False.
+
Raises:
PythonGitError: if something went wrong.
"""
@@ -132,7 +135,10 @@ class PythonGit():
if os.path.exists('setup.py'):
_logger.info('install setup.py for %s' % self.name)
- self._run_pip_install('-e', '.')
+ if dev:
+ self._run_pip_install('-e', '.')
+ else:
+ self._run_pip_install('.')
except PythonGitError as e:
raise e
@@ -185,14 +191,21 @@ dependencies = [
dependencies_dict = {dependency.name: dependency for dependency in dependencies}
-def prepare() -> None:
+def prepare(*args, **kwargs) -> None:
"""
Installs all dependencies from :data:`dependencies` and :data:`parsers`.
"""
for python_git in dependencies:
- python_git.prepare()
+ python_git.prepare(*args, **kwargs)
if __name__ == '__main__':
+ import argparse
+
+ parser = argparse.ArgumentParser(description='Install dependencies from NOMAD-coe.')
+ parser.add_argument('--dev', help='pip install with -e', action='store_true')
+
+ args = parser.parse_args()
+
_logger.setLevel(logging.DEBUG)
- prepare()
+ prepare(dev=args.dev)
--
GitLab