Commit ff577dcd authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch 'v0.8.0-beta' into 'master'

V0.8.0 beta

See merge request !115
parents e0a32f64 c0a86b8e
Pipeline #75282 passed with stages
in 20 minutes and 53 seconds
**/*.git
**/.cache
**/.mypy_cache
**/__pycache__
**/*.pyc
......@@ -7,13 +8,32 @@
.pytest_cache
.vscode/
.volumes/
.git/
.coverage
**/.ipynb_checkpoints
**/.volumes
data/
local/
target/
build/
dist/
dependencies/**/test
dependencies/**/tests
data/
dependencies/**/regtests
dependencies/parsers/phonopy-library/example*
docs/.build
docs/*.graffle
.coverage
examples/
local/
target/
nomad/normalizing/data/*.db
nomad/normalizing/data/*.msg
nomad.yaml
gui/node_modules/
gui/build/
gui/public/metainfo/
gui/npm-debug.log*
gui/yarn-debug.log*
gui/yarn-error.log*
\ No newline at end of file
.DS_Store
.*env/
.pytest/
.python-version
.ipynb_checkpoints/
__pycache__
......@@ -20,5 +21,9 @@ target/
.vscode/
vscode/
nomad.yaml
gunicorn.log.conf
gunicorn.conf
\ No newline at end of file
./gunicorn.log.conf
./gunicorn.conf
build/
dist/
setup.json
parser.osio.log
# default installed image for docker executor is: python:3.6
# using an image that can do git, docker, docker-compose
image: youpy/docker-compose-git
image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/ci-runner
# Uncomment the next lines, to run each pipline/job in its own docker environment.
# Otherwise, it will use the docker of the gitlab runner host (e.g. enc-preprocessing...).
......@@ -12,26 +12,22 @@ image: youpy/docker-compose-git
stages:
- build
- test
- integration
- release
- deploy
- release
variables:
TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:test_${CI_COMMIT_REF_NAME}
RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:${CI_COMMIT_REF_NAME}
TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:${CI_COMMIT_REF_NAME}
STABLE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:stable
LATEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest
FRONTEND_TEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:test_${CI_COMMIT_REF_NAME}
FRONTEND_RELEASE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:${CI_COMMIT_REF_NAME}
FRONTEND_STABLE_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:stable
FRONTEND_LATEST_IMAGE: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/frontend:latest
KUBECONFIG: /etc/deploy/config
build:
stage: build
before_script:
- git submodule sync
- git submodule update --init
- git submodule update --init --jobs=4
# create the version information
- ./gitinfo.sh
script:
# ignore test directories of dependencies, there is a lot of data that we not use
......@@ -44,26 +40,13 @@ build:
- /^dev-.*$/
- tags
buildgui:
stage: build
script:
- cd gui
- ./version.sh
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker build -t $FRONTEND_TEST_IMAGE .
- docker push $FRONTEND_TEST_IMAGE
except:
- /^dev-.*$/
- tags
linting:
stage: test
image: $TEST_IMAGE
script:
- cd /app
- python -m pycodestyle --ignore=E501,E701,E731 nomad tests
- python -m pylint --load-plugins=pylint_mongoengine nomad tests
- python -m pylint --load-plugins=pylint_mongoengine,nomad/metainfo/pylint_plugin nomad tests
- python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests
except:
refs:
......@@ -72,6 +55,14 @@ linting:
- $CI_COMMIT_REF_NAME =~ /^dev-.*$/
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
gui_linting:
stage: test
image: node
script:
- cd gui
- yarn
- yarn run eslint 'src/**/*.js'
tests:
stage: test
image: $TEST_IMAGE
......@@ -91,7 +82,7 @@ tests:
NOMAD_ELASTIC_HOST: elastic
NOMAD_MONGO_HOST: mongo
NOMAD_KEYCLOAK_PASSWORD: ${CI_KEYCLOAK_ADMIN_PASSWORD}
NOMAD_SPRINGER_DB_PATH: /nomad/fairdi/db/data/springer.db
NOMAD_NORMALIZE_SPRINGER_DB_PATH: /nomad/fairdi/db/data/springer.msg
script:
- cd /app
- ls /builds
......@@ -103,6 +94,40 @@ tests:
- $CI_COMMIT_REF_NAME =~ /^dev-.*$/
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
install_tests:
stage: test
image: python:3.6
before_script:
- git submodule sync
- git submodule update --init --jobs=4
script:
- pip install --upgrade pip
- pip install fastentrypoints
- pip install pyyaml
- pip install numpy
- python setup.py compile
- python setup.py sdist
- pip install dist/nomad-0.8.0.tar.gz
- python -c "import nomad.datamodel, nomad.datamodel.metainfo, nomad.client"
- pip install dist/nomad-0.8.0.tar.gz[parsing]
- python -m nomad.cli parse tests/data/parsers/vasp/vasp.xml
deploy:
stage: deploy
before_script:
- mkdir -p /etc/deploy
- echo ${CI_K8S_CONFIG} | base64 -d > ${KUBECONFIG}
script:
- RELEASE_NAME=`echo ${CI_COMMIT_REF_NAME} | sed -e 's/[^A-Za-z0-9\-]/-/g'`
- helm dependency update ops/helm/nomad
- helm upgrade --namespace nomad --install $RELEASE_NAME ops/helm/nomad -f ops/helm/nomad/ci-dev-values.yaml --set proxy.external.path=/dev/nomad/$RELEASE_NAME,image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
- docker pull $TEST_IMAGE
- docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n $CI_DEV_CLUSTER_PROXY/dev/nomad/$RELEASE_NAME/api -u admin -w $CI_NOMAD_ADMIN_PASSWORD integrationtests
except:
- /^dev-.*$/
when: manual
release:
stage: release
script:
......@@ -110,9 +135,7 @@ release:
- docker pull $TEST_IMAGE
- docker tag $TEST_IMAGE $LATEST_IMAGE
- docker push $LATEST_IMAGE
- docker pull $FRONTEND_TEST_IMAGE
- docker tag $FRONTEND_TEST_IMAGE $FRONTEND_LATEST_IMAGE
- docker push $FRONTEND_LATEST_IMAGE
except:
- /^dev-.*$/
when: manual
......@@ -122,35 +145,9 @@ release_version:
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker pull $LATEST_IMAGE
- docker tag $LATEST_IMAGE $RELEASE_IMAGE
- docker tag $LATEST_IMAGE $CI_COMMIT_TAG
- docker tag $LATEST_IMAGE $STABLE_IMAGE
- docker push $RELEASE_IMAGE
- docker push $STABLE_IMAGE
- docker pull $FRONTEND_LATEST_IMAGE
- docker tag $FRONTEND_LATEST_IMAGE $FRONTEND_RELEASE_IMAGE
- docker tag $FRONTEND_LATEST_IMAGE $FRONTEND_STABLE_IMAGE
- docker push $FRONTEND_RELEASE_IMAGE
- docker push $FRONTEND_STABLE_IMAGE
- docker push $CI_COMMIT_TAG
only:
- tags
## TODO This has to be fixed, we need one for testing, staging, production
# deploy_coe_staging:
# stage: deploy
# image: dtzar/helm-kubectl
# before_script:
# - mkdir -p /etc/deploy
# # kube_config is a CI/CD variable set in GitLab GUI
# - echo $CI_KUBE_CONFIG | base64 -d > /etc/deploy/config
# - helm init --upgrade
# - helm repo add stable https://kubernetes-charts.storage.googleapis.com/
# - helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com/
# - helm repo update
# script:
# - cd ops/helm/nomad
# - export KUBECONFIG=/etc/deploy/config
# - helm dep build
# - helm upgrade nomad-coe-staging . --recreate-pods;
# except:
# - /^dev-.*$/
# when: manual
......@@ -81,10 +81,6 @@
path = dependencies/parsers/phonopy
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-phonopy.git
branch = nomad-fair
[submodule "dependencies/parsers/phonopy-library"]
path = dependencies/parsers/phonopy-library
url = https://gitlab.mpcdf.mpg.de/nomad-lab/phonopy.git
branch = nomad-fair
[submodule "dependencies/parsers/gpaw"]
path = dependencies/parsers/gpaw
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-gpaw.git
......@@ -156,3 +152,6 @@
[submodule "dependencies/optimade-python-tools"]
path = dependencies/optimade-python-tools
url = https://github.com/markus1978/optimade-python-tools.git
[submodule "dependencies/parsers/eels"]
path = dependencies/parsers/eels
url = https://github.com/markus1978/eels.git
......@@ -14,8 +14,7 @@
# This dockerfile describes an image that can be used to run the
# - nomad processing worker
# - nomad upload handler that initiates processing after upload
# - nomad api
# - nomad app (incl serving the gui)
# The dockerfile is multistaged to use a fat, more convinient build image and
# copy only necessities to a slim final image
......@@ -23,21 +22,37 @@
# We use slim for the final image
FROM python:3.6-slim as final
# First, build everything in a build image
# First built the GUI in a gui build image
FROM node:latest as gui_build
RUN mkdir -p /app
WORKDIR /app
ENV PATH /app/node_modules/.bin:$PATH
COPY gui/package.json /app/package.json
COPY gui/yarn.lock /app/yarn.lock
RUN yarn
COPY gui /app
RUN yarn run build
# RUN yarn run --silent react-docgen src/components --pretty > react-docgen.out
# Second, build all python stuff in a python build image
FROM python:3.6-stretch as build
RUN mkdir /install
# Install some specific dependencies to make use of docker layer caching
# Install some specific dependencies necessary for the build process
RUN pip install --upgrade pip
RUN pip install fastentrypoints
RUN pip install pyyaml
RUN pip install numpy
# Install some specific dependencies to make use of docker layer caching
RUN pip install cython>=0.19
RUN pip install pandas
RUN pip install h5py
RUN pip install hjson
RUN pip install scipy
RUN pip install scikit-learn==0.20.2
RUN pip install ase==3.15.0
RUN pip install Pint==0.7.2
RUN pip install ase==3.19.0
RUN pip install Pint
RUN pip install matid
RUN pip install mdtraj==1.9.1
RUN pip install mdanalysis==0.16.2
......@@ -46,35 +61,24 @@ RUN pip install mdanalysis==0.16.2
RUN apt-get update && apt-get install -y make
RUN apt-get update && apt-get install -y vim
# We also install the -dev dependencies, to use this image for test and qa
COPY requirements.txt /install/requirements.txt
# Copy files and install nomad@FAIRDI
WORKDIR /install
RUN pip install -r requirements.txt
# Use docker build --build-args CACHEBUST=2 to not cache this (e.g. when you know deps have changed)
ARG CACHEBUST=1
# Install all NOMAD-CoE dependencies. This is done separately because most of
# the time this comes directly from docker cache
COPY ./dependencies /install/dependencies
COPY ./dependencies.sh /install/dependencies.sh
COPY ./.gitmodules /install/.gitmodules
RUN sh dependencies.sh
# Copy rest of files and install nomad@FAIRDI
COPY . /install
RUN pip install .
RUN python setup.py compile
RUN pip install .[all]
RUN python setup.py sdist
WORKDIR /install/docs
# COPY --from=gui_build /app/react-docgen.out /install/docs
RUN make html
RUN \
find /usr/local/lib/python3.6/ -name 'tests' ! -path '*/networkx/*' -exec rm -r '{}' + && \
find /usr/local/lib/python3.6/ -name 'test' -exec rm -r '{}' + && \
find /usr/local/lib/python3.6/site-packages/ -name '*.so' -print -exec sh -c 'file "{}" | grep -q "not stripped" && strip -s "{}"' \;
# Second, create a slim final image
# Third, create a slim final image
FROM final
RUN apt-get update && apt-get install -y --no-install-recommends libgomp1 && apt-get install -y libmagic-dev
RUN apt-get update && apt-get install -y --no-install-recommends libgomp1 && apt-get install -y libmagic-dev curl
# copy the sources for tests, coverage, qa, etc.
COPY . /app
......@@ -88,13 +92,21 @@ RUN echo "copy 2"
# copy the documentation, its files will be served by the API
COPY --from=build /install/docs/.build /app/docs/.build
RUN echo "copy 3"
# copy the nomad command
COPY --from=build /usr/local/bin/nomad /usr/local/bin/nomad
# copy the source distribution, its files will be served by the API
COPY --from=build /install/dist /app/dist
RUN echo "copy 4"
# copy the nomad command
COPY --from=build /usr/local/bin/nomad /usr/bin/nomad
RUN echo "copy 5"
# copy the gui
RUN mkdir -p /app/gui
COPY --from=gui_build /app/build /app/gui/build
RUN echo "copy 6"
RUN mkdir -p /app/.volumes/fs
RUN useradd -ms /bin/bash nomad
RUN chown -R nomad /app
RUN chmod a+rx run.sh
USER nomad
VOLUME /app/.volumes/fs
......
recursive-include dependencies/optimade-python-tools *.txt *.g *.py *.ini
recursive-include nomad *.json *.j2 *.md *.yaml
include README.md
include LICENSE.txt
include requirements.txt
include auto_complete_install.sh
include setup.json
\ No newline at end of file
......@@ -20,6 +20,23 @@ standards like the [Optimade](http://www.optimade.org/) API.
## Getting started
### Using NOMAD as a Python package
You can install the `nomad` Python package from source distribution with pip. Please
note, that this will only install part of NOMAD's dependencies that will only allow
your to use NOMAD's client library, e.g. to access the NOMAD Archive.
```
pip install http://labdev-nomad.esc.rzg.mpg.de/dev/nomad/v0-8-0/dist/nomad-0.8.0
```
To **use the NOMAD parsers for example**, install the `parsing` extra:
```
pip install http://labdev-nomad.esc.rzg.mpg.de/dev/nomad/v0-8-0/dist/nomad-0.8.0[parsing]
nomad parse --show-backend <your-file-to-parse>
```
### For NOMAD developer
Read the [docs](https://repository.nomad-coe.eu/app/docs). The documentation is also part
of the source code. It covers aspects like introduction, architecture, development setup/deployment,
contributing, and API reference.
......@@ -29,6 +46,17 @@ contributing, and API reference.
Omitted versions are plain bugfix releases with only minor changes and fixes.
### v0.8.0
- new underlying datamodel that allows to maintain multiple domains
- mulitple domains supported the GUI
- new metainfo implementation
- API endpoint to access the metainfo
- new archive based on new metainfo
- client library that serves archive data as objects (with tab completion) not dictionaries
- properties and user tab in the search GUI
- improved performance on most parsers
- NOMAD source distribution
### v0.7.9
- Everything to run a simple NOMAD OASIS based on the central user-management
- minor bugfixes
......
#!/usr/bin/env bash
if [[ $(basename $SHELL) = 'bash' ]];
then
if [ -f ~/.bashrc ];
then
echo "Installing bash autocompletion..."
grep -q '_NOMAD_COMPLETE' ~/.bashrc
if [[ $? -ne 0 ]]; then
echo "" >> ~/.bashrc
echo 'eval "$(_NOMAD_COMPLETE=source nomad)"' >> ~/.bashrc
fi
fi
elif [[ $(basename $SHELL) = 'zsh' ]];
then
if [ -f ~/.zshrc ];
then
echo "Installing zsh autocompletion..."
grep -q 'nomad-autocompletion' ~/.zshrc
if [[ $? -ne 0 ]]; then
echo "" >> ~/.zshrc
echo "autoload bashcompinit" >> ~/.zshrc
echo "bashcompinit" >> ~/.zshrc
echo 'eval "$(_NOMAD_COMPLETE=source nomad)"' >> ~/.nomad-autocompletion.sh
echo "source ~/.nomad-autocompletion.sh" >> ~/.zshrc
fi
fi
fi
\ No newline at end of file
#!/bin/sh
set -e
working_dir=`pwd`
echo $working_dir
git config -f .gitmodules --get-regexp '^submodule\..*\.path$' |
while read path_key path
do
cd $working_dir
cd $path
if [ -z "$(git status --porcelain)" ]; then
echo "$path is clean"
else
echo "$path is not clean"
git stash
git checkout nomad-fair-metainfo
git stash apply && git add -A && git commit -a -m "$1"
git push origin nomad-fair-metainfo
fi
done
Subproject commit 487cceae152be217e0689d6217f420480c2a9d39
Subproject commit c424e75671e8c09c2f29c90ec63feafd0a2a706e
Subproject commit df889f4873be90d4a0a3d0b0d3b5fca907e9561c
Subproject commit fe980c53e536f843b6e0ca85642892b8e2f75fed
Subproject commit 95b3874b72035d0529f7dd0cde419192dba68100
Subproject commit 45daba83219defb908044193214acce48baf76e9
Subproject commit 8e6def6b737a195231754f170f6df227096b00c6
Subproject commit 10f066f30370d8b7826d7cc404da360e3b3d92f2
Subproject commit d30ef0bd9275206380866c89946a0c129e7d8df9
Subproject commit f98b920976d5507c9584676e8f77071bcf041cf9
Subproject commit 42b2b0ea894aa83cda9588d3d9bf2cdee1f40a92
Subproject commit e1a03db77aa4acb3f7f55691e11f1277a2ca6726
Subproject commit f9979f05e6a26e0512c26d5a316f4b0749528640
Subproject commit 1cdd522775fa3ad76294bc9a58dbfcaea000f0c5
Subproject commit b87cd29be9fe1637c20a23087ce98855ff1fea44
Subproject commit 5742af2804e002b49bed42a0d5f38f8a79630697
Subproject commit a8b323f8f8b24a5cead4c7e0cca15a78193b12b5
Subproject commit 59a82b9b1218c74660ed570aaa4bdcaa4699c439
Subproject commit 7863212e4bde3443fae8fbb57caf3b0d95fb28f3
Subproject commit a58cfed379d240e5241bb69689441d42a5fb5e0f
Subproject commit de6d49f89748010236baa9b683464b70bc8507a6
Subproject commit 059d4c7f1a452383d67f0ecf0c9bc267580083dc
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment