Refactored docker images.

.dockerignore

@@ -2,10 +2,15 @@
 .pyenv/
 .vscode/
 .volumes/
+.git/
+.mypy_cache/
 data/
 docs/.build
-gui/node_modules/
-gui/build/
+docs/.static
+docs/*.graffle
+gui/
 infrastructure/
 __pycache__/
 *.pyc
+NOMAD.egg-info/
+.coverage

backend.Dockerfile → Dockerfile

@@ -17,49 +17,48 @@
 # - nomad upload handler that initiates processing after upload
 # - nomad api
 
-# The dockerfile is multistaged to help docker with caching unnecessary steps
-# creating a base image with most requirements already installed
-FROM python:3.6-stretch as requirements
+# The dockerfile is multistaged to use a fat, more convinient build image and
+# copy only necessities to a slim final image
+
+# We use slim for the final image
+FROM python:3.6-slim as final
+
+# First, build everything in a build image
+FROM python:3.6-stretch as build
+# Make will be necessary to build the docs with sphynx
+RUN apt-get update && apt-get install -y make
 RUN mkdir /install
 WORKDIR /install
-COPY requirements.txt requirements.txt
+# We also install the -dev dependencies, to use this image for test and qa
+COPY requirements-dev.txt requirements-dev.txt
+RUN pip install -r requirements-dev.txt
 COPY requirements-dep.txt requirements-dep.txt
-RUN pip install -r requirements.txt
 RUN pip install -r requirements-dep.txt
-
-# dependency stage is used to install nomad coe projects
-FROM requirements as dependencies
-WORKDIR /install
-COPY nomad/dependencies.py nomad/dependencies.py
-COPY nomad/config.py nomad/config.py
 COPY requirements.txt requirements.txt
 RUN pip install -r requirements.txt
+# Use docker build --build-args CACHEBUST=2 to not cache this (e.g. when you know deps have changed)
+ARG CACHEBUST=1
+COPY nomad/dependencies.py /install/nomad/dependencies.py
+COPY nomad/config.py /install/nomad/config.py
 RUN python nomad/dependencies.py
+# do that after the dependencies to use docker's layer caching
+COPY . /install
+RUN pip install .
+WORKDIR /install/docs
+RUN make html
 
-# we use slim for the final image
-FROM python:3.6-slim as final
-
-# last stage is used to install the actual code, nomad user, volumes
+# Second, create a slim final image
 FROM final
-# transfer installed packages from dependency stage
-COPY --from=dependencies /usr/local/lib/python3.6/site-packages /usr/local/lib/python3.6/site-packages
-COPY --from=dependencies /usr/local/bin/sphinx-build /usr/local/bin/sphinx-build
-# we also need to copy the install dir, since nomad coe deps are installed with -e
-# TODO that should be changed in production!
-COPY --from=dependencies /install /install
-
-# do stuff
-RUN apt-get update && apt-get install -y make
+# copy the sources for tests, coverage, qa, etc.
 COPY . /app
 WORKDIR /app
-
+# transfer installed packages from dependency stage
+COPY --from=build /usr/local/lib/python3.6/site-packages /usr/local/lib/python3.6/site-packages
 # copy the meta-info, since it files are loaded via relative paths. TODO that should change.
-COPY --from=dependencies /install/.dependencies/nomad-meta-info /app/.dependencies/nomad-meta-info
+COPY --from=build /install/.dependencies/nomad-meta-info /app/.dependencies/nomad-meta-info
+# copy the documentation, its files will be served by the API
+COPY --from=build /install/docs/.build /app/docs/.build
 
-RUN pip install -e .
-WORKDIR /app/docs
-RUN make html
-WORKDIR /app
 RUN useradd -ms /bin/bash nomad
 RUN mkdir -p /app/.volumes/fs; chown -R nomad /app/.volumes/fs
 USER nomad

docs/conf.py

@@ -15,12 +15,17 @@ import os
 import sys
 from recommonmark.transform import AutoStructify
 
-sys.path.insert(0, os.path.abspath('../nomad'))
+sys.path.insert(0, os.path.abspath('..'))
+# TODO, once the normalizers are self contained in their own gits, this should not be
+# necessary anymore
+sys.path.insert(0, os.path.abspath('../.dependencies/normalizers/stats/normalizer/normalizer-stats'))
+sys.path.insert(0, os.path.abspath('../.dependencies/normalizers/symmetry/normalizer/normalizer-symmetry'))
+sys.path.insert(0, os.path.abspath('../.dependencies/normalizers/system-type/normalizer/normalizer-system-type'))
 
 
 # -- Project information -----------------------------------------------------
 
-project = 'NOMAD-XT'
+project = 'nomad-FAIR'
 copyright = '2018, the NOMAD developers'
 author = 'the NOMAD developers'
 
@@ -141,7 +146,7 @@ latex_elements = {
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, 'NOMAD-XT.tex', 'NOMAD-XT Documentation',
+    (master_doc, 'nomad-FAIR.tex', 'nomad-FAIR Documentation',
      'the NOMAD developers', 'manual'),
 ]
 
@@ -151,7 +156,7 @@ latex_documents = [
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    (master_doc, 'nomad', 'NOMAD-XT Documentation',
+    (master_doc, 'nomad', 'nomad-FAIR Documentation',
      [author], 1)
 ]
 
@@ -162,8 +167,8 @@ man_pages = [
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, 'NOMAD-XT', 'NOMAD-XT Documentation',
-     author, 'NOMAD-XT', 'One line description of project.',
+    (master_doc, 'nomad-FAIR', 'nomad-FAIR Documentation',
+     author, 'nomad-FAIR', 'One line description of project.',
      'Miscellaneous'),
 ]
 

frontend.Dockerfile → gui/Dockerfile

@@ -19,19 +19,18 @@
 # intended for the actual GUI container that serves the GUI.
 
 # build environment
-FROM node:latest as builder
+FROM node:latest as build
 RUN mkdir -p /nomad/app
 WORKDIR /nomad/app
 ENV PATH /nomad/app/node_modules/.bin:$PATH
-COPY gui/package.json /nomad/app/package.json
-COPY gui/yarn.lock /nomad/app/yarn.lock
+COPY package.json /nomad/app/package.json
+COPY yarn.lock /nomad/app/yarn.lock
 RUN yarn
-COPY gui /nomad/app
-COPY .git /nomad
+COPY . /nomad/app
 
 RUN yarn build
 
 # production environment
 FROM nginx:1.13.9-alpine
-COPY --from=builder /nomad/app/build /app/nomad
+COPY --from=build /nomad/app/build /app/nomad
 CMD ["nginx", "-g", "daemon off;"]

gui/gitinfo.sh

+#!/bin/sh
+echo \"{ \\\"log\\\": \\\"$(git log -1 --oneline)\\\", \\\"ref\\\": \\\"$(git describe --all)\\\", \\\"version\\\": \\\"$(git describe)\\\" }\"  > src/gitinfo.json
\ No newline at end of file

gui/package.json

@@ -25,8 +25,8 @@
   "scripts": {
     "metainfo": "git clone --single-branch -b nomad-xt http://gitlab.mpcdf.mpg.de/nomad-lab/nomad-meta-info.git --depth=1 public/metainfo",
     "gitinfo": "echo \"{ \\\"log\\\": \\\"$(git log -1 --oneline)\\\", \\\"ref\\\": \\\"$(git describe --all)\\\", \\\"version\\\": \\\"$(git describe)\\\" }\"  > src/gitinfo.json",
-    "start": "yarn metainfo; yarn gitinfo; react-scripts start",
-    "build": "yarn metainfo; yarn gitinfo; react-scripts build",
+    "start": "yarn metainfo; react-scripts start",
+    "build": "yarn metainfo; react-scripts build",
     "test": "react-scripts test --env=jsdom",
     "eject": "react-scripts eject"
   },

nomad/dependencies.py

@@ -95,10 +95,13 @@ class PythonGit():
             raise PythonGitError(
                 'Could not install (pip return code=%s)' % pipcode, repo=self)
 
-    def prepare(self) -> None:
+    def prepare(self, dev: bool = False) -> None:
         """
         Makes sure that the repository is fetched, at the right commit, and installed.
 
+        Arguments:
+            dev (bool): Indicate dev install (uses pip with -e). Default is False.
+
         Raises:
             PythonGitError: if something went wrong.
         """
@@ -132,7 +135,10 @@ class PythonGit():
 
             if os.path.exists('setup.py'):
                 _logger.info('install setup.py for %s' % self.name)
-                self._run_pip_install('-e', '.')
+                if dev:
+                    self._run_pip_install('-e', '.')
+                else:
+                    self._run_pip_install('.')
 
         except PythonGitError as e:
             raise e
@@ -185,14 +191,21 @@ dependencies = [
 dependencies_dict = {dependency.name: dependency for dependency in dependencies}
 
 
-def prepare() -> None:
+def prepare(*args, **kwargs) -> None:
     """
     Installs all dependencies from :data:`dependencies` and :data:`parsers`.
     """
     for python_git in dependencies:
-        python_git.prepare()
+        python_git.prepare(*args, **kwargs)
 
 
 if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser(description='Install dependencies from NOMAD-coe.')
+    parser.add_argument('--dev', help='pip install with -e', action='store_true')
+
+    args = parser.parse_args()
+
     _logger.setLevel(logging.DEBUG)
-    prepare()
+    prepare(dev=args.dev)