Commit e486b276 authored by Markus Scheidgen's avatar Markus Scheidgen

Merge branch 'v0.9.8' into 'master'

v0.9.8 - Merge for release

Closes #445 and #450

See merge request !233
parents 85c7ac01 11888781
Pipeline #90618 passed with stage
in 2 minutes and 48 seconds
......@@ -3,7 +3,7 @@
**/.mypy_cache
**/__pycache__
**/*.pyc
**/NOMAD.egg-info
**/*.egg-info
.*env/
.pyenv*/
.pytest_cache
......@@ -14,6 +14,8 @@
**/.ipynb_checkpoints
**/.volumes
nomad_lab.egg-info/
data/
local/
target/
......@@ -30,6 +32,7 @@ docs/*.graffle
nomad/normalizing/data/*.db
nomad/normalizing/data/*.msg
nomad/app/static
nomad.yaml
gui/node_modules/
......
......@@ -32,4 +32,4 @@ parser.osio.log
gui/src/metainfo.json
gui/src/searchQuantities.json
examples/workdir/
gunicorn.log.conf
......@@ -9,10 +9,11 @@ image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/ci-runner
# services:
# - docker:dind
# TODO remove CI_NOMAD_DEV_PROXY
stages:
- build
- test
- deploy
- release
......@@ -40,7 +41,7 @@ build:
- /^dev-.*$/
- tags
linting:
python linting:
stage: test
image: $TEST_IMAGE
script:
......@@ -55,13 +56,13 @@ linting:
- $CI_COMMIT_REF_NAME =~ /^dev-.*$/
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
gui_linting:
gui linting:
stage: test
image: node
script:
- cd gui
- yarn
- yarn run eslint 'src/**/*.js'
- yarn run lint
except:
refs:
- tags
......@@ -69,7 +70,7 @@ gui_linting:
- $CI_COMMIT_REF_NAME =~ /^dev-.*$/
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
tests:
python tests:
stage: test
image: $TEST_IMAGE
services:
......@@ -100,7 +101,21 @@ tests:
- $CI_COMMIT_REF_NAME =~ /^dev-.*$/
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
install_tests:
gui tests:
stage: test
image: node
script:
- cd gui
- yarn
- yarn run test
except:
refs:
- tags
variables:
- $CI_COMMIT_REF_NAME =~ /^dev-.*$/
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
install tests:
stage: test
image: python:3.7
before_script:
......@@ -124,7 +139,7 @@ install_tests:
- $CI_COMMIT_REF_NAME =~ /^dev-.*$/
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
dev_setup_tests:
dev setup tests:
stage: test
image: python:3.7
script:
......@@ -138,23 +153,36 @@ dev_setup_tests:
- $CI_COMMIT_REF_NAME =~ /^dev-.*$/
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
deploy:
stage: deploy
deploy dev:
stage: release
before_script:
- mkdir -p /etc/deploy
- echo ${CI_K8S_CONFIG} | base64 -d > ${KUBECONFIG}
script:
- RELEASE_NAME=`echo ${CI_COMMIT_REF_NAME} | sed -e 's/[^A-Za-z0-9\-]/-/g'`
- helm dependency update ops/helm/nomad
- helm upgrade --namespace nomad --install $RELEASE_NAME ops/helm/nomad -f ops/helm/nomad/ci-dev-values.yaml --set proxy.external.path=/dev/nomad/$RELEASE_NAME,image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
- helm upgrade --namespace nomad --install $RELEASE_NAME ops/helm/nomad -f ops/helm/nomad/deployments/dev-values.yaml --set proxy.external.path=/dev/nomad/$RELEASE_NAME,image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
- docker pull $TEST_IMAGE
- docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n $CI_DEV_CLUSTER_PROXY/dev/nomad/$RELEASE_NAME/api -u admin -w $CI_NOMAD_ADMIN_PASSWORD integrationtests
- docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n https://nomad-lab.eu/dev/nomad/$RELEASE_NAME/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests --skip-publish --skip-doi
except:
- /^dev-.*$/
when: manual
deploy prod beta:
stage: release
before_script:
- mkdir -p /etc/deploy
- echo ${CI_K8S_PROD_CONFIG} | base64 -d > ${KUBECONFIG}
script:
- helm dependency update ops/helm/nomad
- helm upgrade --install nomad-prod-test ops/helm/nomad -f ops/helm/nomad/deployments/prod-beta-values.yaml --set image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
- docker pull $TEST_IMAGE
- docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n https://nomad-lab.eu/prod/rae/beta/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests --skip-publish --skip-doi
except:
- /^dev-.*$/
when: manual
release:
release latest image:
stage: release
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
......@@ -164,10 +192,8 @@ release:
except:
- /^dev-.*$/
when: manual
only:
- branches
release_version:
release stable image:
stage: release
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
......@@ -176,24 +202,26 @@ release_version:
- docker push $LATEST_IMAGE
- docker tag $TEST_IMAGE $STABLE_IMAGE
- docker push $STABLE_IMAGE
when: manual
only:
- tags
release_pypi:
python package:
stage: release
image: python:3.7
before_script:
- git submodule sync
- git submodule update --init --jobs=4
script:
- pip install --upgrade pip
- pip install fastentrypoints
- pip install pyyaml
- pip install numpy
- pip install twine
- python setup.py compile
- python setup.py sdist
- python -m twine upload -u $CI_TWINE_USER -p $CI_TWINE_PASSWORD dist/*
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker pull $TEST_IMAGE
- docker run --rm $TEST_IMAGE python -m twine upload -u gitlab-ci-token -p ${CI_JOB_TOKEN} --repository-url https://gitlab.mpcdf.mpg.de/api/v4/projects/${CI_PROJECT_ID}/packages/pypi dist/nomad-lab.tar.gz
except:
- /^dev-.*$/
when: manual
pypi package:
stage: release
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker pull $TEST_IMAGE
- docker run --rm $TEST_IMAGE python -m twine upload -u $CI_TWINE_USER -p $CI_TWINE_PASSWORD dist/nomad-lab.tar.gz
when: manual
only:
- tags
......@@ -189,3 +189,6 @@
[submodule "dependencies/parsers/eelsdb"]
path = dependencies/parsers/eelsdb
url = https://gitlab.mpcdf.mpg.de/nomad-lab/eelsdb.git
[submodule "dependencies/parsers/example"]
path = dependencies/parsers/example
url = https://github.com/nomad-coe/nomad-parser-example.git
......@@ -47,7 +47,7 @@ RUN pip install pandas
RUN pip install h5py
RUN pip install hjson
RUN pip install scipy
RUN pip install scikit-learn==0.20.2
RUN pip install scikit-learn
RUN pip install ase==3.19.0
RUN pip install Pint
RUN pip install matid
......@@ -69,8 +69,6 @@ WORKDIR /install
COPY . /install
RUN python setup.py compile
RUN pip install .[all]
RUN python setup.py sdist
RUN cp dist/nomad-lab-*.tar.gz dist/nomad-lab.tar.gz
RUN python -m nomad.cli dev metainfo > gui/src/metainfo.json
RUN python -m nomad.cli dev search-quantities > gui/src/searchQuantities.json
RUN python -m nomad.cli dev toolkit-metadata > gui/src/toolkitMetadata.json
......@@ -98,13 +96,17 @@ COPY --from=build /install/gui/src/toolkitMetadata.json /app/src/toolkitMetadata
COPY --from=build /install/gui/src/units.js /app/src/units.js
RUN yarn run build
# Build the Encyclopedia GUI in the gui build image
# Copy all sources and assets to the GUI build image, build it there, and then
# slim down the contents before they are copied to the final image
RUN mkdir -p /encyclopedia
WORKDIR /encyclopedia
COPY dependencies/encyclopedia-gui/client/src /encyclopedia/src
COPY dependencies/encyclopedia-gui/client/webpack.config.js /encyclopedia/webpack.config.js
COPY dependencies/encyclopedia-gui/client /encyclopedia/
RUN npm install webpack webpack-cli
RUN npx webpack --mode=production
RUN rm -rf /encyclopedia/node_modules
RUN rm -rf /encyclopedia/src
RUN rm -f /encyclopedia/webpack.config.js
RUN rm -f /encyclopedia/.babelrc
# Third, create a slim final image
FROM final
......@@ -115,24 +117,22 @@ COPY . /app
WORKDIR /app
# transfer installed packages from dependency stage
COPY --from=build /usr/local/lib/python3.7/site-packages /usr/local/lib/python3.7/site-packages
RUN echo "copy 1"
# copy the documentation, its files will be served by the API
COPY --from=build /install/docs/.build /app/docs/.build
RUN echo "copy 2"
# copy the source distribution, its files will be served by the API
COPY --from=build /install/dist /app/dist
RUN echo "copy 3"
# copy the nomad command
COPY --from=build /usr/local/bin/nomad /usr/bin/nomad
RUN echo "copy 4"
# copy the gui
RUN mkdir -p /app/gui
COPY --from=gui_build /app/build /app/gui/build
RUN echo "copy 5"
COPY --from=gui_build /app/build /app/nomad/app/static/gui
# copy the encyclopedia gui production code
COPY --from=gui_build /encyclopedia /app/dependencies/encyclopedia-gui/client
RUN rm -rf /app/dependencies/encyclopedia-gui/client/src
RUN echo "copy 6"
COPY --from=gui_build /encyclopedia /app/nomad/app/static/encyclopedia
# remove the developer config on the gui, will be generated by run.sh from nomad.yaml
RUN rm -f /app/nomad/app/static/gui/env.js
RUN rm -f /app/nomad/app/static/encyclopedia/conf.js
# build the python package dist
RUN python setup.py compile
RUN python setup.py sdist
RUN cp dist/nomad-lab-*.tar.gz dist/nomad-lab.tar.gz
RUN mkdir -p /app/.volumes/fs
RUN useradd -ms /bin/bash nomad
......
......@@ -6,3 +6,4 @@ include LICENSE.txt
include requirements.txt
include auto_complete_install.sh
include setup.json
recursive-include nomad/app/static *.css *.ico *.html *.json *.js *.map *.txt *.svg *.png
\ No newline at end of file
......@@ -46,6 +46,9 @@ contributing, and API reference.
Omitted versions are plain bugfix releases with only minor changes and fixes.
### v0.9.8
- A new library for parsing text-based raw files.
### v0.9.3
- Encyclopedia with dedicated materials search index.
......
Subproject commit abe6657cfda6cbc881a0ab8cf5edcc00c2eb8361
Subproject commit 447cb20b5c8fb095bac5861721b8de96502b5058
Subproject commit c17dbecac0f395b48fc65b0c7dcc2ba4de1be271
Subproject commit 9d4518fefb1f1146b7fa7ed95547822683020533
Subproject commit c378ce3c667fa0b6bd93ecdfd5699b7b9e165fa5
Subproject commit 3f78e35b56714fe8c79b010ed52d97def19cb2dc
Subproject commit 3b5866bc8dff77d398eddc6baceb19e1c30180c1
Subproject commit 7ed44cab4410099208f5e399bfa4c1dc2e9c29fe
Subproject commit 8944d5ffd4cac01e43aaac5c5687b0a8831f2bc7
Subproject commit 10973cb74a154c5896633cb50f6e576fddd72852
Subproject commit 56eb84fffa3ae25a4b8dc10aa56da85a4fe6584d
Subproject commit cf6abfe712ff2d0052947f46bdb27131ac6fbaf8
Subproject commit 0bfb17daec156d26110d55cd0918d8fb584f7fac
Subproject commit afdd0937aab2681ca3912cfd9d95c7633fdcd7b9
Subproject commit c1aca04237d69097bbeb17d3a397be66e9c6797f
# API Tutorials
# How to use the APIs
The NOMAD Repository and Archive offers all its functionality through an application
programming interface (API). More specifically a [RESTful HTTP API](https://en.wikipedia.org/wiki/Representational_state_transfer) that allows you
......@@ -38,7 +38,7 @@ Let's say you want to see the repository metadata (i.e. the information that you
our gui) for entries that fit search criteria, like compounds having atoms *Si* and *O* in
it:
```
```sh
curl -X GET "http://nomad-lab.eu/prod/rae/api/repo/?atoms=Si&atoms=O"
```
......@@ -46,7 +46,7 @@ Here we used curl to send an HTTP GET request to return the resource located by
In practice you can omit the `-X GET` (which is the default) and you might want to format
the output:
```
```sh
curl "http://nomad-lab.eu/prod/rae/api/repo/?atoms=Si&atoms=O" | python -m json.tool
```
......@@ -68,21 +68,21 @@ Similar functionality is offered to download archive or raw data. Let's say you
identified an entry (given via a `upload_id`/`calc_id`, see the query output), and
you want to download it:
```
```sh
curl "http://nomad-lab.eu/prod/rae/api/raw/calc/JvdvikbhQp673R4ucwQgiA/k-ckeQ73sflE6GDA80L132VCWp1z/*" -o download.zip
```
With `*` you basically requests all the files under an entry or path..
If you need a specific file (that you already know) of that calculation:
```
```sh
curl "http://nomad-lab.eu/prod/rae/api/raw/calc/JvdvikbhQp673R4ucwQgiA/k-ckeQ73sflE6GDA80L132VCWp1z/INFO.OUT"
```
You can also download a specific file from the upload (given a `upload_id`), if you know
the path of that file:
```
```sh
curl "http://nomad-lab.eu/prod/rae/api/raw/JvdvikbhQp673R4ucwQgiA/exciting_basis_set_error_study/monomers_expanded_k8_rgkmax_080_PBE/72_Hf/INFO.OUT"
```
......@@ -90,27 +90,27 @@ If you have a query
that is more selective, you can also download all results. Here all compounds that only
consist of Si, O, bulk material simulations of cubic systems (currently ~100 entries):
```
```sh
curl "http://nomad-lab.eu/prod/rae/api/raw/query?only_atoms=Si&only_atoms=O&system=bulk&crystal_system=cubic" -o download.zip
```
Here are a few more examples for downloading the raw data of based on DOI or dataset.
You will have to encode non URL safe characters in potential dataset names (e.g. with a service like [www.urlencoder.org](https://www.urlencoder.org/)):
```
```sh
curl "http://nomad-lab.eu/prod/rae/api/raw/query?datasets.doi=10.17172/NOMAD/2020.03.18-1" -o download.zip
curl "http://nomad-lab.eu/prod/rae/api/raw/query?dataset=Full%20ahnarmonic%20stAViC%20approach%3A%20Silicon%20and%20SrTiO3" -o download.zip
```
In a similar way you can see the archive of an entry:
```
```sh
curl "http://nomad-lab.eu/prod/rae/api/archive/f0KQE2aiSz2KRE47QtoZtw/6xe9fZ9xoxBYZOq5lTt8JMgPa3gX" | python -m json.tool
```
Or query and display the first page of 10 archives:
```
```sh
curl "http://nomad-lab.eu/prod/rae/api/archive/query?only_atoms=Si&only_atoms=O" | python -m json.tool
```
......@@ -164,7 +164,7 @@ Optionally, if you need to access your private data, the package *python-keycloa
required to conveniently acquire the necessary tokens to authenticate your self towards
NOMAD.
```
```sh
pip install bravado
pip install python-keycloak
```
......@@ -386,20 +386,20 @@ The shell tool *curl* can be used to call most API endpoints. Most endpoints for
or downloading data are only **GET** operations controlled by URL parameters. For example:
Downloading data:
```
```sh
curl http://nomad-lab.eu/prod/rae/api/raw/query?upload_id=<your_upload_id> -o download.zip
```
It is a litle bit trickier, if you need to authenticate yourself, e.g. to download
not yet published or embargoed data. All endpoints support and most require the use of
an access token. To acquire an access token from our usermanagement system with curl:
```
```sh
curl --data 'grant_type=password&client_id=nomad_public&username=<your_username>&password=<your password>' \
https://nomad-lab.eu/fairdi/keycloak/auth/realms/fairdi_nomad_prod/protocol/openid-connect/token
```
You can use the access-token with:
```
```sh
curl -H 'Authorization: Bearer <you_access_token>' \
http://nomad-lab.eu/prod/rae/api/raw/query?upload_id=<your_upload_id> -o download.zip
```
......
.. _access-the-archive-label:
Using the NOMAD Archive
=======================
Data Access (Archive)
=====================
Of course, you can access the NOMAD Archive directly via the NOMAD API (see the `API tutorial <api_tutorial.html>`_
and `API reference <api.html>`_). But, it is more effective and convenient to use NOMAD's Python client
......
......@@ -11,7 +11,7 @@ are up and running and both have access to the underlying file storage, part of
which is mounted inside each container under :code:`.volumes/fs`.
With both the source and target deployment running, you can use the
:code::ref:`cli_ref:mirror` command to transfer the data from source to target. The
:ref:`cli_ref:mirror` command to transfer the data from source to target. The
mirror will copy everything: i.e. the raw data, archive data and associated
metadata in the database.
......
NOMAD Python package and CLI
-----------------------------------------------------
NOMAD's Python library
----------------------
The :code:`nomad` python package comes with a command line interface (CLI) that
can be accessed after installation by simply running the :code:`nomad` command
in your terminal. The CLI provides a hiearchy of commands by using the `click
......
.. _install-client:
Install the NOMAD client library
================================
......
# Development Setup
## Introduction
The nomad infrastructure consists of a series of nomad and 3rd party services:
- nomad worker (python): task worker that will do the processing
- nomad app (python): the nomad app and it's REST APIs
- nomad gui: a small server serving the web-based react gui
- proxy: an nginx server that reverse proxyies all services under one port
- elastic search: nomad's search and analytics engine
- mongodb: used to store processing state
- rabbitmq: a task queue used to distribute work in a cluster
All 3rd party services should be run via *docker-compose* (see blow). The
nomad python services can be run with python to develop them.
The gui can be run with a development server via yarn.
Below you will find information on how to install all python dependencies and code
manually. How to use *docker*/*docker-compose*. How run 3rd-party services with *docker-compose*.
Keep in mind the *docker-compose* configures all services in a way that mirror
the configuration of the python code in `nomad/config.py` and the gui config in
`gui/.env.development`.
To learn about how to run everything in docker, e.g. to operate a NOMAD OASIS in
production, go (here)(/app/docs/ops.html).
## Install python code and dependencies
### Cloning and development tools
If not already done, you should clone nomad and create a python virtual environment.
To clone the repository:
```
git clone git@gitlab.mpcdf.mpg.de:nomad-lab/nomad-FAIR.git
cd nomad-FAIR
```
### C libs
Even though the NOMAD infrastructure is written in python, there is a C library
required by one of our python dependencies.
#### libmagic
Libmagic allows to determine the MIME type of files. It should be installed on most
unix/linux systems. It can be installed on MacOS with homebrew:
```
brew install libmagic
```
### Virtual environment
#### pyenv
The nomad code currently targets python 3.7. If you host machine has an older version installed,
you can use [pyenv](https://github.com/pyenv/pyenv) to use python 3.7 in parallel to your
system's python.
#### virtualenv
We strongly recommend to use *virtualenv* to create a virtual environment. It will allow you
to keep nomad and its dependencies separate from your system's python installation.
Make sure to base the virtual environment on Python 3.
To install *virtualenv*, create an environment and activate the environment use:
```
pip install virtualenv
virtualenv -p `which python3` .pyenv
source .pyenv/bin/activate
```
#### Conda
If you are a conda user, there is an equivalent, but you have to install pip and the
right python version while creating the environment.
```
conda create --name nomad_env pip python=3.7
conda activate nomad_env
```
To install libmagick for conda, you can use (other channels might also work):
```
conda install -c conda-forge --name nomad_env libmagic
```
#### pip
Make sure you have the most recent version of pip:
```
pip install --upgrade pip
```
The next steps can be done using the `setup.sh` script. If you prefer to understand all
the steps and run them manually, read on:
### Install NOMAD-coe dependencies.
Nomad is based on python modules from the NOMAD-coe project.
This includes parsers, python-common and the meta-info. These modules are maintained as
their own GITLab/git repositories. To clone and initialize them run:
```
git submodule update --init
```
All requirements for these submodules need to be installed and they need to be installed
themselves as python modules. Run the `dependencies.sh` script that will install
everything into your virtual environment:
```
./dependencies.sh -e
```
The `-e` option will install the NOMAD-coe dependencies with symbolic links allowing you
to change the downloaded dependency code without having to reinstall after.
### Install nomad
Finally, you can add nomad to the environment itself (including all extras)
```
pip install -e .[all]
```
If pip tries to use and compile sources and this creates errors, it can be told to prefer binary version:
```
pip install -e .[all] --prefer-binary
```
### Generate GUI artifacts
The NOMAD GUI requires static artifacts that are generated from the NOMAD Python codes.
```
nomad dev metainfo > gui/src/metainfo.json
nomad dev searchQuantities > gui/src/searchQuantities.json
nomad dev units > gui/src/units.js
./gitinfo.sh
```
In additional, you have to do some more steps to prepare your working copy to run all
the tests. See below.
## Build and run the infrastructure with docker
### Docker and nomad
Nomad depends on a set of databases, search engines, and other services. Those