Commit e486b276 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch 'v0.9.8' into 'master'

v0.9.8 - Merge for release

Closes #445 and #450

See merge request !233
parents 85c7ac01 11888781
Pipeline #90618 passed with stage
in 2 minutes and 48 seconds
......@@ -3,7 +3,7 @@
**/.mypy_cache
**/__pycache__
**/*.pyc
**/NOMAD.egg-info
**/*.egg-info
.*env/
.pyenv*/
.pytest_cache
......@@ -14,6 +14,8 @@
**/.ipynb_checkpoints
**/.volumes
nomad_lab.egg-info/
data/
local/
target/
......@@ -30,6 +32,7 @@ docs/*.graffle
nomad/normalizing/data/*.db
nomad/normalizing/data/*.msg
nomad/app/static
nomad.yaml
gui/node_modules/
......
......@@ -32,4 +32,4 @@ parser.osio.log
gui/src/metainfo.json
gui/src/searchQuantities.json
examples/workdir/
gunicorn.log.conf
......@@ -9,10 +9,11 @@ image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/ci-runner
# services:
# - docker:dind
# TODO remove CI_NOMAD_DEV_PROXY
stages:
- build
- test
- deploy
- release
......@@ -40,7 +41,7 @@ build:
- /^dev-.*$/
- tags
linting:
python linting:
stage: test
image: $TEST_IMAGE
script:
......@@ -55,13 +56,13 @@ linting:
- $CI_COMMIT_REF_NAME =~ /^dev-.*$/
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
gui_linting:
gui linting:
stage: test
image: node
script:
- cd gui
- yarn
- yarn run eslint 'src/**/*.js'
- yarn run lint
except:
refs:
- tags
......@@ -69,7 +70,7 @@ gui_linting:
- $CI_COMMIT_REF_NAME =~ /^dev-.*$/
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
tests:
python tests:
stage: test
image: $TEST_IMAGE
services:
......@@ -100,7 +101,21 @@ tests:
- $CI_COMMIT_REF_NAME =~ /^dev-.*$/
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
install_tests:
gui tests:
stage: test
image: node
script:
- cd gui
- yarn
- yarn run test
except:
refs:
- tags
variables:
- $CI_COMMIT_REF_NAME =~ /^dev-.*$/
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
install tests:
stage: test
image: python:3.7
before_script:
......@@ -124,7 +139,7 @@ install_tests:
- $CI_COMMIT_REF_NAME =~ /^dev-.*$/
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
dev_setup_tests:
dev setup tests:
stage: test
image: python:3.7
script:
......@@ -138,23 +153,36 @@ dev_setup_tests:
- $CI_COMMIT_REF_NAME =~ /^dev-.*$/
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
deploy:
stage: deploy
deploy dev:
stage: release
before_script:
- mkdir -p /etc/deploy
- echo ${CI_K8S_CONFIG} | base64 -d > ${KUBECONFIG}
script:
- RELEASE_NAME=`echo ${CI_COMMIT_REF_NAME} | sed -e 's/[^A-Za-z0-9\-]/-/g'`
- helm dependency update ops/helm/nomad
- helm upgrade --namespace nomad --install $RELEASE_NAME ops/helm/nomad -f ops/helm/nomad/ci-dev-values.yaml --set proxy.external.path=/dev/nomad/$RELEASE_NAME,image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
- helm upgrade --namespace nomad --install $RELEASE_NAME ops/helm/nomad -f ops/helm/nomad/deployments/dev-values.yaml --set proxy.external.path=/dev/nomad/$RELEASE_NAME,image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
- docker pull $TEST_IMAGE
- docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n $CI_DEV_CLUSTER_PROXY/dev/nomad/$RELEASE_NAME/api -u admin -w $CI_NOMAD_ADMIN_PASSWORD integrationtests
- docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n https://nomad-lab.eu/dev/nomad/$RELEASE_NAME/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests --skip-publish --skip-doi
except:
- /^dev-.*$/
when: manual
deploy prod beta:
stage: release
before_script:
- mkdir -p /etc/deploy
- echo ${CI_K8S_PROD_CONFIG} | base64 -d > ${KUBECONFIG}
script:
- helm dependency update ops/helm/nomad
- helm upgrade --install nomad-prod-test ops/helm/nomad -f ops/helm/nomad/deployments/prod-beta-values.yaml --set image.tag=$CI_COMMIT_REF_NAME,roll=true --wait
- docker pull $TEST_IMAGE
- docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod $TEST_IMAGE python -m nomad.cli client -n https://nomad-lab.eu/prod/rae/beta/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests --skip-publish --skip-doi
except:
- /^dev-.*$/
when: manual
release:
release latest image:
stage: release
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
......@@ -164,10 +192,8 @@ release:
except:
- /^dev-.*$/
when: manual
only:
- branches
release_version:
release stable image:
stage: release
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
......@@ -176,24 +202,26 @@ release_version:
- docker push $LATEST_IMAGE
- docker tag $TEST_IMAGE $STABLE_IMAGE
- docker push $STABLE_IMAGE
when: manual
only:
- tags
release_pypi:
python package:
stage: release
image: python:3.7
before_script:
- git submodule sync
- git submodule update --init --jobs=4
script:
- pip install --upgrade pip
- pip install fastentrypoints
- pip install pyyaml
- pip install numpy
- pip install twine
- python setup.py compile
- python setup.py sdist
- python -m twine upload -u $CI_TWINE_USER -p $CI_TWINE_PASSWORD dist/*
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker pull $TEST_IMAGE
- docker run --rm $TEST_IMAGE python -m twine upload -u gitlab-ci-token -p ${CI_JOB_TOKEN} --repository-url https://gitlab.mpcdf.mpg.de/api/v4/projects/${CI_PROJECT_ID}/packages/pypi dist/nomad-lab.tar.gz
except:
- /^dev-.*$/
when: manual
pypi package:
stage: release
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
- docker pull $TEST_IMAGE
- docker run --rm $TEST_IMAGE python -m twine upload -u $CI_TWINE_USER -p $CI_TWINE_PASSWORD dist/nomad-lab.tar.gz
when: manual
only:
- tags
......@@ -189,3 +189,6 @@
[submodule "dependencies/parsers/eelsdb"]
path = dependencies/parsers/eelsdb
url = https://gitlab.mpcdf.mpg.de/nomad-lab/eelsdb.git
[submodule "dependencies/parsers/example"]
path = dependencies/parsers/example
url = https://github.com/nomad-coe/nomad-parser-example.git
......@@ -47,7 +47,7 @@ RUN pip install pandas
RUN pip install h5py
RUN pip install hjson
RUN pip install scipy
RUN pip install scikit-learn==0.20.2
RUN pip install scikit-learn
RUN pip install ase==3.19.0
RUN pip install Pint
RUN pip install matid
......@@ -69,8 +69,6 @@ WORKDIR /install
COPY . /install
RUN python setup.py compile
RUN pip install .[all]
RUN python setup.py sdist
RUN cp dist/nomad-lab-*.tar.gz dist/nomad-lab.tar.gz
RUN python -m nomad.cli dev metainfo > gui/src/metainfo.json
RUN python -m nomad.cli dev search-quantities > gui/src/searchQuantities.json
RUN python -m nomad.cli dev toolkit-metadata > gui/src/toolkitMetadata.json
......@@ -98,13 +96,17 @@ COPY --from=build /install/gui/src/toolkitMetadata.json /app/src/toolkitMetadata
COPY --from=build /install/gui/src/units.js /app/src/units.js
RUN yarn run build
# Build the Encyclopedia GUI in the gui build image
# Copy all sources and assets to the GUI build image, build it there, and then
# slim down the contents before they are copied to the final image
RUN mkdir -p /encyclopedia
WORKDIR /encyclopedia
COPY dependencies/encyclopedia-gui/client/src /encyclopedia/src
COPY dependencies/encyclopedia-gui/client/webpack.config.js /encyclopedia/webpack.config.js
COPY dependencies/encyclopedia-gui/client /encyclopedia/
RUN npm install webpack webpack-cli
RUN npx webpack --mode=production
RUN rm -rf /encyclopedia/node_modules
RUN rm -rf /encyclopedia/src
RUN rm -f /encyclopedia/webpack.config.js
RUN rm -f /encyclopedia/.babelrc
# Third, create a slim final image
FROM final
......@@ -115,24 +117,22 @@ COPY . /app
WORKDIR /app
# transfer installed packages from dependency stage
COPY --from=build /usr/local/lib/python3.7/site-packages /usr/local/lib/python3.7/site-packages
RUN echo "copy 1"
# copy the documentation, its files will be served by the API
COPY --from=build /install/docs/.build /app/docs/.build
RUN echo "copy 2"
# copy the source distribution, its files will be served by the API
COPY --from=build /install/dist /app/dist
RUN echo "copy 3"
# copy the nomad command
COPY --from=build /usr/local/bin/nomad /usr/bin/nomad
RUN echo "copy 4"
# copy the gui
RUN mkdir -p /app/gui
COPY --from=gui_build /app/build /app/gui/build
RUN echo "copy 5"
COPY --from=gui_build /app/build /app/nomad/app/static/gui
# copy the encyclopedia gui production code
COPY --from=gui_build /encyclopedia /app/dependencies/encyclopedia-gui/client
RUN rm -rf /app/dependencies/encyclopedia-gui/client/src
RUN echo "copy 6"
COPY --from=gui_build /encyclopedia /app/nomad/app/static/encyclopedia
# remove the developer config on the gui, will be generated by run.sh from nomad.yaml
RUN rm -f /app/nomad/app/static/gui/env.js
RUN rm -f /app/nomad/app/static/encyclopedia/conf.js
# build the python package dist
RUN python setup.py compile
RUN python setup.py sdist
RUN cp dist/nomad-lab-*.tar.gz dist/nomad-lab.tar.gz
RUN mkdir -p /app/.volumes/fs
RUN useradd -ms /bin/bash nomad
......
......@@ -6,3 +6,4 @@ include LICENSE.txt
include requirements.txt
include auto_complete_install.sh
include setup.json
recursive-include nomad/app/static *.css *.ico *.html *.json *.js *.map *.txt *.svg *.png
\ No newline at end of file
......@@ -46,6 +46,9 @@ contributing, and API reference.
Omitted versions are plain bugfix releases with only minor changes and fixes.
### v0.9.8
- A new library for parsing text-based raw files.
### v0.9.3
- Encyclopedia with dedicated materials search index.
......
Subproject commit abe6657cfda6cbc881a0ab8cf5edcc00c2eb8361
Subproject commit 447cb20b5c8fb095bac5861721b8de96502b5058
Subproject commit c17dbecac0f395b48fc65b0c7dcc2ba4de1be271
Subproject commit 9d4518fefb1f1146b7fa7ed95547822683020533
Subproject commit c378ce3c667fa0b6bd93ecdfd5699b7b9e165fa5
Subproject commit 3f78e35b56714fe8c79b010ed52d97def19cb2dc
Subproject commit 3b5866bc8dff77d398eddc6baceb19e1c30180c1
Subproject commit 7ed44cab4410099208f5e399bfa4c1dc2e9c29fe
Subproject commit 8944d5ffd4cac01e43aaac5c5687b0a8831f2bc7
Subproject commit 10973cb74a154c5896633cb50f6e576fddd72852
Subproject commit 56eb84fffa3ae25a4b8dc10aa56da85a4fe6584d
Subproject commit cf6abfe712ff2d0052947f46bdb27131ac6fbaf8
Subproject commit 0bfb17daec156d26110d55cd0918d8fb584f7fac
Subproject commit afdd0937aab2681ca3912cfd9d95c7633fdcd7b9
Subproject commit c1aca04237d69097bbeb17d3a397be66e9c6797f
# API Tutorials
# How to use the APIs
The NOMAD Repository and Archive offers all its functionality through an application
programming interface (API). More specifically a [RESTful HTTP API](https://en.wikipedia.org/wiki/Representational_state_transfer) that allows you
......@@ -38,7 +38,7 @@ Let's say you want to see the repository metadata (i.e. the information that you
our gui) for entries that fit search criteria, like compounds having atoms *Si* and *O* in
it:
```
```sh
curl -X GET "http://nomad-lab.eu/prod/rae/api/repo/?atoms=Si&atoms=O"
```
......@@ -46,7 +46,7 @@ Here we used curl to send an HTTP GET request to return the resource located by
In practice you can omit the `-X GET` (which is the default) and you might want to format
the output:
```
```sh
curl "http://nomad-lab.eu/prod/rae/api/repo/?atoms=Si&atoms=O" | python -m json.tool
```
......@@ -68,21 +68,21 @@ Similar functionality is offered to download archive or raw data. Let's say you
identified an entry (given via a `upload_id`/`calc_id`, see the query output), and
you want to download it:
```
```sh
curl "http://nomad-lab.eu/prod/rae/api/raw/calc/JvdvikbhQp673R4ucwQgiA/k-ckeQ73sflE6GDA80L132VCWp1z/*" -o download.zip
```
With `*` you basically requests all the files under an entry or path..
If you need a specific file (that you already know) of that calculation:
```
```sh
curl "http://nomad-lab.eu/prod/rae/api/raw/calc/JvdvikbhQp673R4ucwQgiA/k-ckeQ73sflE6GDA80L132VCWp1z/INFO.OUT"
```
You can also download a specific file from the upload (given a `upload_id`), if you know
the path of that file:
```
```sh
curl "http://nomad-lab.eu/prod/rae/api/raw/JvdvikbhQp673R4ucwQgiA/exciting_basis_set_error_study/monomers_expanded_k8_rgkmax_080_PBE/72_Hf/INFO.OUT"
```
......@@ -90,27 +90,27 @@ If you have a query
that is more selective, you can also download all results. Here all compounds that only
consist of Si, O, bulk material simulations of cubic systems (currently ~100 entries):
```
```sh
curl "http://nomad-lab.eu/prod/rae/api/raw/query?only_atoms=Si&only_atoms=O&system=bulk&crystal_system=cubic" -o download.zip
```
Here are a few more examples for downloading the raw data of based on DOI or dataset.
You will have to encode non URL safe characters in potential dataset names (e.g. with a service like [www.urlencoder.org](https://www.urlencoder.org/)):
```
```sh
curl "http://nomad-lab.eu/prod/rae/api/raw/query?datasets.doi=10.17172/NOMAD/2020.03.18-1" -o download.zip
curl "http://nomad-lab.eu/prod/rae/api/raw/query?dataset=Full%20ahnarmonic%20stAViC%20approach%3A%20Silicon%20and%20SrTiO3" -o download.zip
```
In a similar way you can see the archive of an entry:
```
```sh
curl "http://nomad-lab.eu/prod/rae/api/archive/f0KQE2aiSz2KRE47QtoZtw/6xe9fZ9xoxBYZOq5lTt8JMgPa3gX" | python -m json.tool
```
Or query and display the first page of 10 archives:
```
```sh
curl "http://nomad-lab.eu/prod/rae/api/archive/query?only_atoms=Si&only_atoms=O" | python -m json.tool
```
......@@ -164,7 +164,7 @@ Optionally, if you need to access your private data, the package *python-keycloa
required to conveniently acquire the necessary tokens to authenticate your self towards
NOMAD.
```
```sh
pip install bravado
pip install python-keycloak
```
......@@ -386,20 +386,20 @@ The shell tool *curl* can be used to call most API endpoints. Most endpoints for
or downloading data are only **GET** operations controlled by URL parameters. For example:
Downloading data:
```
```sh
curl http://nomad-lab.eu/prod/rae/api/raw/query?upload_id=<your_upload_id> -o download.zip
```
It is a litle bit trickier, if you need to authenticate yourself, e.g. to download
not yet published or embargoed data. All endpoints support and most require the use of
an access token. To acquire an access token from our usermanagement system with curl:
```
```sh
curl --data 'grant_type=password&client_id=nomad_public&username=<your_username>&password=<your password>' \
https://nomad-lab.eu/fairdi/keycloak/auth/realms/fairdi_nomad_prod/protocol/openid-connect/token
```
You can use the access-token with:
```
```sh
curl -H 'Authorization: Bearer <you_access_token>' \
http://nomad-lab.eu/prod/rae/api/raw/query?upload_id=<your_upload_id> -o download.zip
```
......
.. _access-the-archive-label:
Using the NOMAD Archive
=======================
Data Access (Archive)
=====================
Of course, you can access the NOMAD Archive directly via the NOMAD API (see the `API tutorial <api_tutorial.html>`_
and `API reference <api.html>`_). But, it is more effective and convenient to use NOMAD's Python client
......
......@@ -11,7 +11,7 @@ are up and running and both have access to the underlying file storage, part of
which is mounted inside each container under :code:`.volumes/fs`.
With both the source and target deployment running, you can use the
:code::ref:`cli_ref:mirror` command to transfer the data from source to target. The
:ref:`cli_ref:mirror` command to transfer the data from source to target. The
mirror will copy everything: i.e. the raw data, archive data and associated
metadata in the database.
......
NOMAD Python package and CLI
-----------------------------------------------------
NOMAD's Python library
----------------------
The :code:`nomad` python package comes with a command line interface (CLI) that
can be accessed after installation by simply running the :code:`nomad` command
in your terminal. The CLI provides a hiearchy of commands by using the `click
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment