Commit 1ff41c24 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Refactored docker images.

parent 96dbea05
......@@ -2,10 +2,15 @@
.pyenv/
.vscode/
.volumes/
.git/
.mypy_cache/
data/
docs/.build
gui/node_modules/
gui/build/
docs/.static
docs/*.graffle
gui/
infrastructure/
__pycache__/
*.pyc
NOMAD.egg-info/
.coverage
......@@ -17,49 +17,48 @@
# - nomad upload handler that initiates processing after upload
# - nomad api
# The dockerfile is multistaged to help docker with caching unnecessary steps
# creating a base image with most requirements already installed
FROM python:3.6-stretch as requirements
# The dockerfile is multistaged to use a fat, more convinient build image and
# copy only necessities to a slim final image
# We use slim for the final image
FROM python:3.6-slim as final
# First, build everything in a build image
FROM python:3.6-stretch as build
# Make will be necessary to build the docs with sphynx
RUN apt-get update && apt-get install -y make
RUN mkdir /install
WORKDIR /install
COPY requirements.txt requirements.txt
# We also install the -dev dependencies, to use this image for test and qa
COPY requirements-dev.txt requirements-dev.txt
RUN pip install -r requirements-dev.txt
COPY requirements-dep.txt requirements-dep.txt
RUN pip install -r requirements.txt
RUN pip install -r requirements-dep.txt
# dependency stage is used to install nomad coe projects
FROM requirements as dependencies
WORKDIR /install
COPY nomad/dependencies.py nomad/dependencies.py
COPY nomad/config.py nomad/config.py
COPY requirements.txt requirements.txt
RUN pip install -r requirements.txt
# Use docker build --build-args CACHEBUST=2 to not cache this (e.g. when you know deps have changed)
ARG CACHEBUST=1
COPY nomad/dependencies.py /install/nomad/dependencies.py
COPY nomad/config.py /install/nomad/config.py
RUN python nomad/dependencies.py
# do that after the dependencies to use docker's layer caching
COPY . /install
RUN pip install .
WORKDIR /install/docs
RUN make html
# we use slim for the final image
FROM python:3.6-slim as final
# last stage is used to install the actual code, nomad user, volumes
# Second, create a slim final image
FROM final
# transfer installed packages from dependency stage
COPY --from=dependencies /usr/local/lib/python3.6/site-packages /usr/local/lib/python3.6/site-packages
COPY --from=dependencies /usr/local/bin/sphinx-build /usr/local/bin/sphinx-build
# we also need to copy the install dir, since nomad coe deps are installed with -e
# TODO that should be changed in production!
COPY --from=dependencies /install /install
# do stuff
RUN apt-get update && apt-get install -y make
# copy the sources for tests, coverage, qa, etc.
COPY . /app
WORKDIR /app
# transfer installed packages from dependency stage
COPY --from=build /usr/local/lib/python3.6/site-packages /usr/local/lib/python3.6/site-packages
# copy the meta-info, since it files are loaded via relative paths. TODO that should change.
COPY --from=dependencies /install/.dependencies/nomad-meta-info /app/.dependencies/nomad-meta-info
COPY --from=build /install/.dependencies/nomad-meta-info /app/.dependencies/nomad-meta-info
# copy the documentation, its files will be served by the API
COPY --from=build /install/docs/.build /app/docs/.build
RUN pip install -e .
WORKDIR /app/docs
RUN make html
WORKDIR /app
RUN useradd -ms /bin/bash nomad
RUN mkdir -p /app/.volumes/fs; chown -R nomad /app/.volumes/fs
USER nomad
......
......@@ -15,12 +15,17 @@ import os
import sys
from recommonmark.transform import AutoStructify
sys.path.insert(0, os.path.abspath('../nomad'))
sys.path.insert(0, os.path.abspath('..'))
# TODO, once the normalizers are self contained in their own gits, this should not be
# necessary anymore
sys.path.insert(0, os.path.abspath('../.dependencies/normalizers/stats/normalizer/normalizer-stats'))
sys.path.insert(0, os.path.abspath('../.dependencies/normalizers/symmetry/normalizer/normalizer-symmetry'))
sys.path.insert(0, os.path.abspath('../.dependencies/normalizers/system-type/normalizer/normalizer-system-type'))
# -- Project information -----------------------------------------------------
project = 'NOMAD-XT'
project = 'nomad-FAIR'
copyright = '2018, the NOMAD developers'
author = 'the NOMAD developers'
......@@ -141,7 +146,7 @@ latex_elements = {
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
(master_doc, 'NOMAD-XT.tex', 'NOMAD-XT Documentation',
(master_doc, 'nomad-FAIR.tex', 'nomad-FAIR Documentation',
'the NOMAD developers', 'manual'),
]
......@@ -151,7 +156,7 @@ latex_documents = [
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
(master_doc, 'nomad', 'NOMAD-XT Documentation',
(master_doc, 'nomad', 'nomad-FAIR Documentation',
[author], 1)
]
......@@ -162,8 +167,8 @@ man_pages = [
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
(master_doc, 'NOMAD-XT', 'NOMAD-XT Documentation',
author, 'NOMAD-XT', 'One line description of project.',
(master_doc, 'nomad-FAIR', 'nomad-FAIR Documentation',
author, 'nomad-FAIR', 'One line description of project.',
'Miscellaneous'),
]
......
......@@ -19,19 +19,18 @@
# intended for the actual GUI container that serves the GUI.
# build environment
FROM node:latest as builder
FROM node:latest as build
RUN mkdir -p /nomad/app
WORKDIR /nomad/app
ENV PATH /nomad/app/node_modules/.bin:$PATH
COPY gui/package.json /nomad/app/package.json
COPY gui/yarn.lock /nomad/app/yarn.lock
COPY package.json /nomad/app/package.json
COPY yarn.lock /nomad/app/yarn.lock
RUN yarn
COPY gui /nomad/app
COPY .git /nomad
COPY . /nomad/app
RUN yarn build
# production environment
FROM nginx:1.13.9-alpine
COPY --from=builder /nomad/app/build /app/nomad
COPY --from=build /nomad/app/build /app/nomad
CMD ["nginx", "-g", "daemon off;"]
#!/bin/sh
echo \"{ \\\"log\\\": \\\"$(git log -1 --oneline)\\\", \\\"ref\\\": \\\"$(git describe --all)\\\", \\\"version\\\": \\\"$(git describe)\\\" }\" > src/gitinfo.json
\ No newline at end of file
......@@ -25,8 +25,8 @@
"scripts": {
"metainfo": "git clone --single-branch -b nomad-xt http://gitlab.mpcdf.mpg.de/nomad-lab/nomad-meta-info.git --depth=1 public/metainfo",
"gitinfo": "echo \"{ \\\"log\\\": \\\"$(git log -1 --oneline)\\\", \\\"ref\\\": \\\"$(git describe --all)\\\", \\\"version\\\": \\\"$(git describe)\\\" }\" > src/gitinfo.json",
"start": "yarn metainfo; yarn gitinfo; react-scripts start",
"build": "yarn metainfo; yarn gitinfo; react-scripts build",
"start": "yarn metainfo; react-scripts start",
"build": "yarn metainfo; react-scripts build",
"test": "react-scripts test --env=jsdom",
"eject": "react-scripts eject"
},
......
......@@ -95,10 +95,13 @@ class PythonGit():
raise PythonGitError(
'Could not install (pip return code=%s)' % pipcode, repo=self)
def prepare(self) -> None:
def prepare(self, dev: bool = False) -> None:
"""
Makes sure that the repository is fetched, at the right commit, and installed.
Arguments:
dev (bool): Indicate dev install (uses pip with -e). Default is False.
Raises:
PythonGitError: if something went wrong.
"""
......@@ -132,7 +135,10 @@ class PythonGit():
if os.path.exists('setup.py'):
_logger.info('install setup.py for %s' % self.name)
self._run_pip_install('-e', '.')
if dev:
self._run_pip_install('-e', '.')
else:
self._run_pip_install('.')
except PythonGitError as e:
raise e
......@@ -185,14 +191,21 @@ dependencies = [
dependencies_dict = {dependency.name: dependency for dependency in dependencies}
def prepare() -> None:
def prepare(*args, **kwargs) -> None:
"""
Installs all dependencies from :data:`dependencies` and :data:`parsers`.
"""
for python_git in dependencies:
python_git.prepare()
python_git.prepare(*args, **kwargs)
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser(description='Install dependencies from NOMAD-coe.')
parser.add_argument('--dev', help='pip install with -e', action='store_true')
args = parser.parse_args()
_logger.setLevel(logging.DEBUG)
prepare()
prepare(dev=args.dev)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment