Refactored the setup.py. Using compiled arguments from deps, layers of extras....

Refactored the setup.py. Using compiled arguments from deps, layers of extras. Some import refactoring to make the extra layers work.

.dockerignore

@@ -16,6 +16,8 @@ local/
 target/
 examples/
 ops/
+build/
+dist/
 
 dependencies/**/test
 dependencies/**/tests

.gitignore

@@ -23,3 +23,6 @@ vscode/
 nomad.yaml
 ./gunicorn.log.conf
 ./gunicorn.conf
+build/
+dist/
+setup.json
\ No newline at end of file

.gitmodules

@@ -81,10 +81,6 @@
 	path = dependencies/parsers/phonopy
 	url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-phonopy.git
 	branch = nomad-fair
-[submodule "dependencies/parsers/phonopy-library"]
-	path = dependencies/parsers/phonopy-library
-	url = https://gitlab.mpcdf.mpg.de/nomad-lab/phonopy.git
-	branch = nomad-fair
 [submodule "dependencies/parsers/gpaw"]
 	path = dependencies/parsers/gpaw
 	url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-gpaw.git

Dockerfile

@@ -14,8 +14,7 @@
 
 # This dockerfile describes an image that can be used to run the
 # - nomad processing worker
-# - nomad upload handler that initiates processing after upload
-# - nomad api
+# - nomad app (incl serving the gui)
 
 # The dockerfile is multistaged to use a fat, more convinient build image and
 # copy only necessities to a slim final image
@@ -27,16 +26,20 @@ FROM python:3.6-slim as final
 FROM python:3.6-stretch as build
 RUN mkdir /install
 
-# Install some specific dependencies to make use of docker layer caching
+# Install some specific dependencies necessary for the build process
 RUN pip install --upgrade pip
+RUN pip install fastentrypoints
+RUN pip install pyyaml
 RUN pip install numpy
+
+# Install some specific dependencies to make use of docker layer caching
 RUN pip install cython>=0.19
 RUN pip install pandas
 RUN pip install h5py
 RUN pip install hjson
 RUN pip install scipy
 RUN pip install scikit-learn==0.20.2
-RUN pip install ase==3.15.0
+RUN pip install ase==3.19.0
 RUN pip install Pint
 RUN pip install matid
 RUN pip install mdtraj==1.9.1
@@ -46,24 +49,12 @@ RUN pip install mdanalysis==0.16.2
 RUN apt-get update && apt-get install -y make
 RUN apt-get update && apt-get install -y vim
 
-# We also install the -dev dependencies, to use this image for test and qa
-COPY requirements.txt /install/requirements.txt
+# Copy files and install nomad@FAIRDI
 WORKDIR /install
-RUN pip install -r requirements.txt
-
-# Use docker build --build-args CACHEBUST=2 to not cache this (e.g. when you know deps have changed)
-ARG CACHEBUST=1
-
-# Install all NOMAD-CoE dependencies. This is done separately because most of
-# the time this comes directly from docker cache
-COPY ./dependencies /install/dependencies
-COPY ./dependencies.sh /install/dependencies.sh
-COPY ./.gitmodules /install/.gitmodules
-RUN sh dependencies.sh
-
-# Copy rest of files and install nomad@FAIRDI
 COPY . /install
-RUN pip install .
+RUN python setup.py compile
+RUN pip install .[all]
+RUN python setup.py sdist
 WORKDIR /install/docs
 RUN make html
 RUN \
@@ -99,13 +90,16 @@ RUN echo "copy 2"
 # copy the documentation, its files will be served by the API
 COPY --from=build /install/docs/.build /app/docs/.build
 RUN echo "copy 3"
+# copy the source distribution, its files will be served by the API
+COPY --from=build /install/dist /app/dist
+RUN echo "copy 4"
 # copy the nomad command
 COPY --from=build /usr/local/bin/nomad /usr/bin/nomad
-RUN echo "copy 4"
+RUN echo "copy 5"
 # copy the gui
 RUN mkdir -p /app/gui
 COPY --from=gui_build /app/build /app/gui/build
-RUN echo "copy 5"
+RUN echo "copy 6"
 
 RUN mkdir -p /app/.volumes/fs
 RUN useradd -ms /bin/bash nomad

MANIFEST.in

+recursive-include dependencies/optimade-python-tools *.txt *.g *.py *.ini
+recursive-include nomad *.json *.j2 *.md *.yaml
+include README.md
+include requirements.txt
+include auto_complete_install.sh
+include setup.json
\ No newline at end of file

cpmd @ 59a82b9b

-Subproject commit e3a0ab05c403a97155ffc8bdc1a9c8321b1c4278
+Subproject commit 59a82b9b1218c74660ed570aaa4bdcaa4699c439

dl-poly @ 059d4c7f

-Subproject commit 7f905180fdbcdd4e82f11a98419364f200298d51
+Subproject commit 059d4c7f1a452383d67f0ecf0c9bc267580083dc

phonopy @ 09bc6105

-Subproject commit eac05cd4c80498e8e6ccd8f959cc76d9e16bcb2e
+Subproject commit 09bc61058470381a9d590e70cf9ee0c76fb120f2

phonopy-library @ dcc29f74

-Subproject commit dcc29f7418dd193c53e8815bdbe20a32ccaf37ea

python_common @ d04f6964

-Subproject commit f0350d256e6d15d9ab089f6bb0ce9c87b5c42243
+Subproject commit d04f69646ce811c92d8b3ea274d9c9afee9996b2

docs/setup.md

@@ -85,11 +85,6 @@ conda -c conda-forge install --name nomad_env libmagic
 The next steps can be done using the `setup.sh` script. If you prefere to understand all
 the steps and run them manually, read on:
 
-### Install python dependencies
-We use *pip* to manage required python packages.
-```
-pip install -r requirements.txt
-```
 
 ### Install NOMAD-coe dependencies.
 Nomad is based on python modules from the NOMAD-coe project.
@@ -111,9 +106,9 @@ The `-e` option will install the NOMAD-coe dependencies with symbolic links allo
 to change the downloaded dependency code without having to reinstall after.
 
 ### Install nomad
-Finally, you can add nomad to the environment itself.
+Finally, you can add nomad to the environment itself (including all extras)
 ```
-pip install -e .
+pip install -e .[all]
 ```
 
 ## Build and run the infrastructure with docker

examples/client.py

+from nomad import config
+from nomad.client import query_archive
+from nomad.metainfo import units
+
+# this will not be necessary, once this is the official NOMAD version
+config.client.url = 'http://labdev-nomad.esc.rzg.mpg.de/fairdi/nomad/testing-major/api'
+
+
+aq = query_archive(
+    query={
+        'upload_id': ['b5rGMO6dT4Gzqn3JaLjPpw']
+    },
+    required={
+        'section_run': {
+            'section_single_configuration_calculation[0]': {
+                'energy_total': '*'
+            }
+        }
+    },
+    per_page=100, max=1000)
+
+print('total', aq.total)
+
+for i, e in enumerate(aq):
+    if i % 200 == 0:
+        print(e.section_run[0].section_single_configuration_calculation[0].energy_total)
+
+print(aq)

examples/files.py

 from nomad import infrastructure, files, processing as proc
 
-infrastructure.setup_logging()
 infrastructure.setup_mongo()
 
 upload_id = 'NvVyk3gATxCJW6dWS4cRWw'

examples/parse.py

 import sys
-
-from nomad import utils
 from nomad.cli.parse import parse
 
-
-utils.configure_logging()
 parse(sys.argv[1])

examples/search.py

@@ -5,7 +5,6 @@ config.elastic.host = 'localhost'
 config.elastic.port = 19202
 config.elastic.index_name = 'fairdi_nomad_prod_v0_7'
 
-infrastructure.setup_logging()
 infrastructure.setup_elastic()
 
 

gui/package.json

 {
   "name": "nomad-fair-gui",
   "version": "0.8.0",
-  "commit": "nomad-gui-commit-placeholder",
+  "commit": "e98694e",
   "private": true,
   "dependencies": {
     "@material-ui/core": "^3.9.3",

nomad/app/__init__.py

@@ -34,6 +34,7 @@ from nomad import config, utils as nomad_utils
 from .api import blueprint as api_blueprint, api
 from .optimade import blueprint as optimade_blueprint, api as optimade
 from .docs import blueprint as docs_blueprint
+from .dist import blueprint as dist_blueprint
 from .gui import blueprint as gui_blueprint
 from . import common
 
@@ -92,6 +93,7 @@ CORS(app)
 app.register_blueprint(api_blueprint, url_prefix='/api')
 app.register_blueprint(optimade_blueprint, url_prefix='/optimade')
 app.register_blueprint(docs_blueprint, url_prefix='/docs')
+app.register_blueprint(dist_blueprint, url_prefix='/dist')
 app.register_blueprint(gui_blueprint, url_prefix='/gui')
 
 

nomad/app/dist.py

+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from flask import Blueprint
+import os.path
+
+dist_folder = os.path.abspath(os.path.join(
+    os.path.dirname(__file__), '../../dist'))
+blueprint = Blueprint('dist', __name__, static_url_path='/', static_folder=dist_folder)

nomad/cli/__init__.py

@@ -19,13 +19,14 @@ that offers various functionality to the command line user.
 Use it from the command line with ``nomad --help`` or ``python -m nomad.cli --help`` to learn
 more.
 '''
-import lazy_import
+from nomad.cli import lazy_import
 
 lazy_import.lazy_module('click')
 lazy_import.lazy_module('logging')
 lazy_import.lazy_module('os')
 lazy_import.lazy_module('typing')
 lazy_import.lazy_module('json')
+lazy_import.lazy_module('orjson')
 lazy_import.lazy_module('sys')
 lazy_import.lazy_module('nomad.config')
 lazy_import.lazy_module('nomad.infrastructure')
@@ -41,8 +42,4 @@ lazy_import.lazy_module('nomadcore')
 lazy_import.lazy_module('nomadcore.simple_parser')
 
 from . import dev, parse, admin, client  # noqa
-from .cli import cli  # noqa
-
-
-def run_cli():
-    cli()  # pylint: disable=E1120,E1123
+from .cli import run_cli, cli  # noqa

nomad/cli/__main__.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .cli import cli
+from .cli import run_cli
 
 if __name__ == '__main__':
-    cli()  # pylint: disable=E1120,E1123
+    run_cli()  # pylint: disable=E1120,E1123