Commit ee3e4e87 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Refactored the setup.py. Using compiled arguments from deps, layers of extras....

Refactored the setup.py. Using compiled arguments from deps, layers of extras. Some import refactoring to make the extra layers work.
parent a66ad806
Pipeline #72627 failed with stages
in 23 minutes and 25 seconds
......@@ -16,6 +16,8 @@ local/
target/
examples/
ops/
build/
dist/
dependencies/**/test
dependencies/**/tests
......
......@@ -23,3 +23,6 @@ vscode/
nomad.yaml
./gunicorn.log.conf
./gunicorn.conf
build/
dist/
setup.json
\ No newline at end of file
......@@ -81,10 +81,6 @@
path = dependencies/parsers/phonopy
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-phonopy.git
branch = nomad-fair
[submodule "dependencies/parsers/phonopy-library"]
path = dependencies/parsers/phonopy-library
url = https://gitlab.mpcdf.mpg.de/nomad-lab/phonopy.git
branch = nomad-fair
[submodule "dependencies/parsers/gpaw"]
path = dependencies/parsers/gpaw
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-gpaw.git
......
......@@ -14,8 +14,7 @@
# This dockerfile describes an image that can be used to run the
# - nomad processing worker
# - nomad upload handler that initiates processing after upload
# - nomad api
# - nomad app (incl serving the gui)
# The dockerfile is multistaged to use a fat, more convinient build image and
# copy only necessities to a slim final image
......@@ -27,16 +26,20 @@ FROM python:3.6-slim as final
FROM python:3.6-stretch as build
RUN mkdir /install
# Install some specific dependencies to make use of docker layer caching
# Install some specific dependencies necessary for the build process
RUN pip install --upgrade pip
RUN pip install fastentrypoints
RUN pip install pyyaml
RUN pip install numpy
# Install some specific dependencies to make use of docker layer caching
RUN pip install cython>=0.19
RUN pip install pandas
RUN pip install h5py
RUN pip install hjson
RUN pip install scipy
RUN pip install scikit-learn==0.20.2
RUN pip install ase==3.15.0
RUN pip install ase==3.19.0
RUN pip install Pint
RUN pip install matid
RUN pip install mdtraj==1.9.1
......@@ -46,24 +49,12 @@ RUN pip install mdanalysis==0.16.2
RUN apt-get update && apt-get install -y make
RUN apt-get update && apt-get install -y vim
# We also install the -dev dependencies, to use this image for test and qa
COPY requirements.txt /install/requirements.txt
# Copy files and install nomad@FAIRDI
WORKDIR /install
RUN pip install -r requirements.txt
# Use docker build --build-args CACHEBUST=2 to not cache this (e.g. when you know deps have changed)
ARG CACHEBUST=1
# Install all NOMAD-CoE dependencies. This is done separately because most of
# the time this comes directly from docker cache
COPY ./dependencies /install/dependencies
COPY ./dependencies.sh /install/dependencies.sh
COPY ./.gitmodules /install/.gitmodules
RUN sh dependencies.sh
# Copy rest of files and install nomad@FAIRDI
COPY . /install
RUN pip install .
RUN python setup.py compile
RUN pip install .[all]
RUN python setup.py sdist
WORKDIR /install/docs
RUN make html
RUN \
......@@ -99,13 +90,16 @@ RUN echo "copy 2"
# copy the documentation, its files will be served by the API
COPY --from=build /install/docs/.build /app/docs/.build
RUN echo "copy 3"
# copy the source distribution, its files will be served by the API
COPY --from=build /install/dist /app/dist
RUN echo "copy 4"
# copy the nomad command
COPY --from=build /usr/local/bin/nomad /usr/bin/nomad
RUN echo "copy 4"
RUN echo "copy 5"
# copy the gui
RUN mkdir -p /app/gui
COPY --from=gui_build /app/build /app/gui/build
RUN echo "copy 5"
RUN echo "copy 6"
RUN mkdir -p /app/.volumes/fs
RUN useradd -ms /bin/bash nomad
......
recursive-include dependencies/optimade-python-tools *.txt *.g *.py *.ini
recursive-include nomad *.json *.j2 *.md *.yaml
include README.md
include requirements.txt
include auto_complete_install.sh
include setup.json
\ No newline at end of file
Subproject commit e3a0ab05c403a97155ffc8bdc1a9c8321b1c4278
Subproject commit 59a82b9b1218c74660ed570aaa4bdcaa4699c439
Subproject commit 7f905180fdbcdd4e82f11a98419364f200298d51
Subproject commit 059d4c7f1a452383d67f0ecf0c9bc267580083dc
Subproject commit eac05cd4c80498e8e6ccd8f959cc76d9e16bcb2e
Subproject commit 09bc61058470381a9d590e70cf9ee0c76fb120f2
Subproject commit dcc29f7418dd193c53e8815bdbe20a32ccaf37ea
Subproject commit f0350d256e6d15d9ab089f6bb0ce9c87b5c42243
Subproject commit d04f69646ce811c92d8b3ea274d9c9afee9996b2
......@@ -85,11 +85,6 @@ conda -c conda-forge install --name nomad_env libmagic
The next steps can be done using the `setup.sh` script. If you prefere to understand all
the steps and run them manually, read on:
### Install python dependencies
We use *pip* to manage required python packages.
```
pip install -r requirements.txt
```
### Install NOMAD-coe dependencies.
Nomad is based on python modules from the NOMAD-coe project.
......@@ -111,9 +106,9 @@ The `-e` option will install the NOMAD-coe dependencies with symbolic links allo
to change the downloaded dependency code without having to reinstall after.
### Install nomad
Finally, you can add nomad to the environment itself.
Finally, you can add nomad to the environment itself (including all extras)
```
pip install -e .
pip install -e .[all]
```
## Build and run the infrastructure with docker
......
from nomad import config
from nomad.client import query_archive
from nomad.metainfo import units
# this will not be necessary, once this is the official NOMAD version
config.client.url = 'http://labdev-nomad.esc.rzg.mpg.de/fairdi/nomad/testing-major/api'
aq = query_archive(
query={
'upload_id': ['b5rGMO6dT4Gzqn3JaLjPpw']
},
required={
'section_run': {
'section_single_configuration_calculation[0]': {
'energy_total': '*'
}
}
},
per_page=100, max=1000)
print('total', aq.total)
for i, e in enumerate(aq):
if i % 200 == 0:
print(e.section_run[0].section_single_configuration_calculation[0].energy_total)
print(aq)
from nomad import infrastructure, files, processing as proc
infrastructure.setup_logging()
infrastructure.setup_mongo()
upload_id = 'NvVyk3gATxCJW6dWS4cRWw'
......
import sys
from nomad import utils
from nomad.cli.parse import parse
utils.configure_logging()
parse(sys.argv[1])
......@@ -5,7 +5,6 @@ config.elastic.host = 'localhost'
config.elastic.port = 19202
config.elastic.index_name = 'fairdi_nomad_prod_v0_7'
infrastructure.setup_logging()
infrastructure.setup_elastic()
......
{
"name": "nomad-fair-gui",
"version": "0.8.0",
"commit": "nomad-gui-commit-placeholder",
"commit": "e98694e",
"private": true,
"dependencies": {
"@material-ui/core": "^3.9.3",
......
......@@ -34,6 +34,7 @@ from nomad import config, utils as nomad_utils
from .api import blueprint as api_blueprint, api
from .optimade import blueprint as optimade_blueprint, api as optimade
from .docs import blueprint as docs_blueprint
from .dist import blueprint as dist_blueprint
from .gui import blueprint as gui_blueprint
from . import common
......@@ -92,6 +93,7 @@ CORS(app)
app.register_blueprint(api_blueprint, url_prefix='/api')
app.register_blueprint(optimade_blueprint, url_prefix='/optimade')
app.register_blueprint(docs_blueprint, url_prefix='/docs')
app.register_blueprint(dist_blueprint, url_prefix='/dist')
app.register_blueprint(gui_blueprint, url_prefix='/gui')
......
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from flask import Blueprint
import os.path
dist_folder = os.path.abspath(os.path.join(
os.path.dirname(__file__), '../../dist'))
blueprint = Blueprint('dist', __name__, static_url_path='/', static_folder=dist_folder)
......@@ -19,13 +19,14 @@ that offers various functionality to the command line user.
Use it from the command line with ``nomad --help`` or ``python -m nomad.cli --help`` to learn
more.
'''
import lazy_import
from nomad.cli import lazy_import
lazy_import.lazy_module('click')
lazy_import.lazy_module('logging')
lazy_import.lazy_module('os')
lazy_import.lazy_module('typing')
lazy_import.lazy_module('json')
lazy_import.lazy_module('orjson')
lazy_import.lazy_module('sys')
lazy_import.lazy_module('nomad.config')
lazy_import.lazy_module('nomad.infrastructure')
......@@ -41,8 +42,4 @@ lazy_import.lazy_module('nomadcore')
lazy_import.lazy_module('nomadcore.simple_parser')
from . import dev, parse, admin, client # noqa
from .cli import cli # noqa
def run_cli():
cli() # pylint: disable=E1120,E1123
from .cli import run_cli, cli # noqa
......@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from .cli import cli
from .cli import run_cli
if __name__ == '__main__':
cli() # pylint: disable=E1120,E1123
run_cli() # pylint: disable=E1120,E1123
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment