Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • nomad-lab/nomad-FAIR
  • pgoel/nomad-FAIR
  • jpd47/nomad-FAIR
3 results
Show changes
Commits on Source (28)
Showing with 317 additions and 1814 deletions
......@@ -36,6 +36,9 @@ gui/public/env.js
!gui/tests/nomad.yaml
!ops/docker-compose/nomad-oasis/configs/nomad.yaml
!ops/docker-compose/nomad-oasis-with-keycloak/configs/nomad.yaml
!ops/docker-compose/nomad-oasis-with-plugins/nomad.yaml
!examples/plugins/schema/nomad.yaml
!examples/plugins/parser/nomad.yaml
# Ignore built gui and docs artufacts
nomad/app/static/
......
......@@ -18,6 +18,7 @@ gui/junit.xml
# Ignore all user-specific configuration files
nomad.yaml
gui/public/env.js
gui/public/artifacts.js
!gui/tests/nomad.yaml
!ops/docker-compose/nomad-oasis/configs/nomad.yaml
!ops/docker-compose/nomad-oasis-with-keycloak/configs/nomad.yaml
......
......@@ -33,3 +33,9 @@
[submodule "dependencies/matid"]
path = dependencies/matid
url = https://github.com/SINGROUP/matid.git
[submodule "examples/plugins/schema"]
path = examples/plugins/schema
url = https://github.com/nomad-coe/nomad-schema-plugin-example.git
[submodule "examples/plugins/parser"]
path = examples/plugins/parser
url = https://github.com/nomad-coe/nomad-parser-plugin-example.git
......@@ -28,7 +28,6 @@ disable=blacklisted-name,
empty-docstring,
unneeded-not,
singleton-comparison,
misplaced-comparison-constant,
unidiomatic-typecheck,
consider-using-enumerate,
consider-iterating-dictionary,
......@@ -57,12 +56,10 @@ disable=blacklisted-name,
wrong-import-position,
useless-import-alias,
import-outside-toplevel,
old-style-class,
len-as-condition,
raw-checker-failed,
bad-inline-option,
locally-disabled,
locally-enabled,
file-ignored,
suppressed-message,
useless-suppression,
......@@ -111,7 +108,6 @@ disable=blacklisted-name,
expression-not-assigned,
unnecessary-pass,
unnecessary-lambda,
deprecated-lambda,
assign-to-new-keyword,
useless-else-on-loop,
exec-used,
......@@ -133,15 +129,12 @@ disable=blacklisted-name,
useless-super-delegation,
bad-indentation,
mixed-indentation,
lowercase-l-suffix,
wildcard-import,
deprecated-module,
relative-import,
reimported,
import-self,
misplaced-future,
fixme,
invalid-encoded-data,
global-variable-undefined,
global-statement,
global-at-module-level,
......@@ -149,7 +142,6 @@ disable=blacklisted-name,
unused-wildcard-import,
redefined-outer-name,
redefined-builtin,
redefine-in-handler,
undefined-loop-variable,
cell-var-from-loop,
possibly-unused-variable,
......@@ -158,9 +150,7 @@ disable=blacklisted-name,
broad-except,
duplicate-except,
try-except-raise,
nonstandard-exception,
raising-format-tuple,
property-on-old-class,
keyword-arg-before-vararg,
logging-not-lazy,
logging-format-interpolation,
......@@ -219,7 +209,6 @@ disable=blacklisted-name,
range-builtin-not-iterating,
filter-builtin-not-iterating,
using-cmp-argument,
eq-without-hash,
div-method,
idiv-method,
rdiv-method,
......@@ -313,10 +302,7 @@ enable=syntax-error,
raising-non-exception,
notimplemented-raised,
catching-non-exception,
slots-on-old-class,
super-on-old-class,
bad-super-call,
missing-super-argument,
no-member,
not-callable,
assignment-from-no-return,
......@@ -358,11 +344,7 @@ enable=syntax-error,
unpacking-in-except,
old-raise-syntax,
backtick,
long-suffix,
old-ne-operator,
old-octal-literal,
import-star-module-level,
non-ascii-bytes-literal,
yield-inside-async-function,
not-async-context-manager,
fatal,
......
......@@ -87,6 +87,18 @@
"tests/data/parsers/vasp/vasp.xml"
]
},
{
"name": "nomad gui generate",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"module": "nomad.cli",
"justMyCode": false,
"args": [
"dev",
"gui-artifacts"
]
},
{
"name": "Python: Current File",
"type": "python",
......@@ -116,7 +128,7 @@
"directory": "gui"
}
],
"python.testing.pytestPath": ".pyenv/bin/pytest",
"python.testing.pytestPath": "pytest",
"python.testing.pytestArgs": [
"tests"
],
......
......@@ -48,8 +48,9 @@ RUN yarn --network-timeout 1200000
# Artifact for running the tests
COPY tests/states/archives/dft.json /app/tests/states/archives/dft.json
# Copy and build the appticaion itself
# Copy and build the applicaion itself
COPY gui .
RUN echo "REACT_APP_BACKEND_URL=/fairdi/nomad/latest" > .env
RUN yarn run build
......@@ -111,17 +112,9 @@ COPY .pylintrc \
# Files requiered for artifact generation/testing
COPY ops/docker-compose ./ops/docker-compose
COPY gui/src/metainfo.json ./gui/src/metainfo.json
COPY gui/src/searchQuantities.json ./gui/src/searchQuantities.json
COPY gui/src/toolkitMetadata.json ./gui/src/toolkitMetadata.json
COPY gui/src/unitsData.js ./gui/src/unitsData.js
COPY gui/src/parserMetadata.json ./gui/src/parserMetadata.json
COPY dependencies/nomad-remote-tools-hub/tools.json ./dependencies/nomad-remote-tools-hub/tools.json
COPY gui/src/northTools.json ./gui/src/northTools.json
COPY gui/src/exampleUploads.json ./gui/src/exampleUploads.json
COPY gui/tests/nomad.yaml ./gui/tests/nomad.yaml
COPY gui/tests/env.js ./gui/tests/env.js
COPY gui/tests/artifacts.js ./gui/tests/artifacts.js
# build the example upload files
RUN ./scripts/generate_example_uploads.sh
......@@ -228,4 +221,6 @@ USER nomad
EXPOSE 8000
EXPOSE 9000
ENV PYTHONPATH=/app/plugins
VOLUME /app/.volumes/fs
prune *
exclude *
recursive-include nomad *.py *.json *.j2 *.md *.txt metadata.yaml
recursive-include nomad *.py *.json *.j2 *.md *.txt nomad_plugin.yaml
include pyproject.toml setup.py AUTHORS LICENSE README.md README.parsers.md requirements.txt requirements-dev.txt
graft nomad/app/static
exclude nomad/app/static/gui/env.js
exclude nomad/app/static/gui/artifact.js
include dependencies/matid/README.md
recursive-include dependencies/matid/matid *.py
include dependencies/nomad-dos-fingerprints/README.md
recursive-include dependencies/nomad-dos-fingerprints/nomad_dos_fingerprints *.py
include dependencies/parsers/atomistic/README.md
recursive-include dependencies/parsers/atomistic/atomisticparsers *.py metadata.yaml
recursive-include dependencies/parsers/atomistic/atomisticparsers *.py nomad_plugin.yaml
include dependencies/parsers/database/README.md
recursive-include dependencies/parsers/database/databaseparsers *.py metadata.yaml
recursive-include dependencies/parsers/database/databaseparsers *.py nomad_plugin.yaml
include dependencies/parsers/eelsdb/README.md
recursive-include dependencies/parsers/eelsdb/eelsdbparser *.py metadata.yaml
recursive-include dependencies/parsers/eelsdb/eelsdbparser *.py nomad_plugin.yaml
include dependencies/parsers/electronic/README.md
recursive-include dependencies/parsers/electronic/electronicparsers *.py metadata.yaml
recursive-include dependencies/parsers/electronic/electronicparsers *.py nomad_plugin.yaml
include dependencies/parsers/nexus/README.md
recursive-include dependencies/parsers/nexus/nexusutils *.py
include dependencies/parsers/nexus/nexusutils/dataconverter/readers/hall/enum_map.json
recursive-include dependencies/parsers/nexus/nexusutils/definitions *.xml *.xsd
include dependencies/parsers/workflow/README.md
recursive-include dependencies/parsers/workflow/workflowparsers *.py metadata.yaml
recursive-include dependencies/parsers/workflow/workflowparsers *.py nomad_plugin.yaml
Subproject commit 262e5e4153e0ec33e41634eb8d87470b8a1a044f
Subproject commit 3e9d4c2b3fd21663fcc2049860b675d34d3dac8b
Subproject commit fa700636ab4cba6349deb99ffa9af816c912986f
Subproject commit ae2576f89c6e4aafd3de8044869b0631c115985d
Subproject commit 47d42e4d6818d0333460ce1ff9649c739a128d71
Subproject commit 217f826a94ae709cfa541a5586f1b4e82221a57e
Subproject commit aabcd0182d8d16f6d927ea29556f40280cf68580
Subproject commit ad90f31de79620893b67d38deb469c4cc138550a
Subproject commit 8a4e81f81c1870847fc2cf73551fddc512bb39bb
Subproject commit e7fa508ed8ead8d40687db3676a3c3abc37129d2
Subproject commit 5d1022ae5302011345f58c1c4026cc0cdd4e7907
Subproject commit 58f7fa362c288bec9a971684d92c2370647d2171
......@@ -110,51 +110,63 @@ for Intel, and install and use an Intel based Python. The second answer in this
describes how to use both the Apple and Intel homebrew simultaneously.
### Install nomad
The following command can be used to install all dependencies of all submodules and nomad
itself. If successful you can skip the rest of this *Install nomad* section.
The following command can be used to install all dependencies of all submodules
and nomad itself.
```
./scripts/setup_dev_env.sh
```
Install all the requirements needed for development (including submodul requirements):
??? note "Installation details"
```sh
pip install --prefer-binary -r requirements-dev.txt
```
Here is more detailed rundown of the installation steps.
Finally, you can add nomad to the environment itself (including all extras).
The `-e` option will install the NOMAD with symbolic links allowing you
to change the code without having to reinstall after each change.
```sh
pip install -e .[parsing,infrastructure,dev]
```
First we ensure that all submodules are up-to-date:
If pip tries to use and compile sources and this creates errors, it can be told to prefer binary version:
```sh
git submodule update --init --recursive
```
```sh
pip install -e .[parsing,infrastructure,dev] --prefer-binary
```
Previous build is cleaned:
```sh
rm -rf nomad/app/static/docs
rm -rf nomad/app/static/gui
rm -rf site
```
### Update GUI artifacts
All the requirements needed for development (including submodule requirements) are installed:
The NOMAD GUI requires static artifacts that are generated from the NOMAD Python codes.
```sh
python -m nomad.cli dev gui-artifacts --output-directory gui/src
python -m nomad.cli dev gui-config >gui/public/env.js
```
```sh
pip install --prefer-binary -r requirements-dev.txt
```
Or simply run
```sh
./scripts/generate_gui_artifacts.sh
```
Next we install the `nomad` package itself (including all extras). The `-e`
option will install the NOMAD with symbolic links allowing you to change
the code without having to reinstall after each change.
```sh
pip install -e .[parsing,infrastructure,dev]
```
If pip tries to use and compile sources and this creates errors, it can be told to prefer binary version:
```sh
pip install -e .[parsing,infrastructure,dev] --prefer-binary
```
The NOMAD GUI requires a static .env file that can be generated with:
The generated files are stored in GIT. The GUI code might not match the expected data in
outdated files. If there are changes to units, metainfo, new parsers, new toolkit notebooks it
might be necessary to regenerate these gui artifacts.
```sh
python -m nomad.cli dev gui-env > gui/.env.development
```
In addition, you have to do some more steps to prepare your working copy to run all
the tests. See below.
This file includes some of the server details that are needed so that the
GUI can make the initial connection properly. If you e.g. change the server
address in your NOMAD configuration file, it will be necessary to regenerate
this .env file. In production this file will be overridden.
In addition, you have to do some more steps to prepare your working copy to run
all the tests. See below.
## Run the infrastructure
......@@ -212,15 +224,28 @@ via your preferred tools. Just make sure to use the right ports.
## Run NOMAD
### nomad.yaml
Before you run NOMAD for development purposes, you should configure it to use the `test`
realm of our user management system. By default, NOMAD will use the `fairdi_nomad_prod` realm.
Create a `nomad.yaml` file in the root folder:
```
```yaml
keycloak:
realm_name: fairdi_nomad_test
```
You might also want to exclude some of the default plugins, or only include the plugins
you'll need. Especially plugins that slower start-up and import times due to instantiation
of large schemas (e.g. nexus create couple thousand definitions for 70+ applications) can
often be excluded.
```yaml
plugins:
exclude:
- parsers/nexus
```
### App and Worker
NOMAD consist of the NOMAD app/api, a worker, and the GUI. You can run the app and the worker with
the NOMAD cli. These commands will run the services and display their log output. You should open
......@@ -249,16 +274,11 @@ To run the worker directly with celery, do (from the root)
celery -A nomad.processing worker -l info
```
Before you can run the gui, make sure that generated artifacts are up-to-date:
Before you can run the gui, make sure that generated the GUI artifacts:
```sh
./scripts/generate_gui_artifacts.sh
```
Also, make sure that the config file (`gui/public/env.js`) have been properly created:
```sh
./scripts/generate_gui_config.sh
```
If you run the gui on its own (e.g. with react dev server below), you also have to have
the app manually. The gui and its dependencies run on [node](https://nodejs.org) and
the [yarn](https://yarnpkg.com/) dependency manager. Read their documentation on how to
......@@ -366,6 +386,8 @@ of testing.
to using an actual browser (does not support e.g. WebGL), but in practice
is realistic enough for the majority of the test.
#### Test structure
We have adopted a `pytest`-like structure for organizing the test utilities:
each source code folder may contain a `conftest.js` file that contains
utilities that are relevant for testing the code in that particular folder.
......@@ -446,44 +468,54 @@ def search():
)
data.save()
```
When running in the `test-integration` or `test-record` mode (see below), this
function will be executed in order to prepare the application backend. The
`closeAPI` function will handle cleaning the test state between successive
`startAPI` calls: it will completely wipe out MongoDB, ElasticSearch and the
upload files.
!!! note
When running in the online mode (see below), this function will be executed in
order to prepare the application backend. The `closeAPI` function will handle
cleaning the test state between successive `startAPI` calls: it will completely
wipe out MongoDB, ElasticSearch and the upload files.
The tests are using the configuration specified in `gui/tests/nomad.yaml`, that
specifies a separate database/filesystem config in order to prevent interacting with
any other instances of NOMAD.
#### Running tests
The tests can be run in two different modes. _Offline testing_ uses
pre-recorded files to mock the API traffic during testing. This allows one to
run tests more quickly without a server. During _online testing_, the tests
perform calls to a running server where a test state has been prepared. This
mode can be used to perform integration tests, but also to record the snapshot
files needed by the offline testing.
In order to control how the API traffic is handled, there are three main ways
for running the test suite, as configured in `package.json`:
##### Offline testing
This is the way our CI pipeline runs the tests and should be used locally
whenever you wish to e.g. reproduce pipeline errors or when your tests do not
involve any API traffic.
- `yarn test [<filename>]`: Runs the tests parallelly in an 'offline'
mode: `startAPI` will use pre-recorded API snapshot files that are found in
gui/tests.
- `yarn test-integration filename>]`: Runs the tests serially and `startAPI`
will forward any API traffic to a live API that is running locally.
- `yarn test-record [<filename>]`: Runs the tests serially and `startAPI` will
forward traffic to a live API that is running locally, additionally
recording the traffic to the specified snapshot file.
1. Ensure that the gui artifacts are up-to-date:
```sh
./scripts/generate_gui_test_artifacts.sh
```
As snapshot tests do not connect to the server, the artifacts cannot be
fetched dynamically from the server and static files need to be used
instead.
!!! note
2. Run `yarn test` to run the whole suite or `yarn test [<filename>]` to run a
specific test.
##### Online testing
When you wish to record API traffic for offline testing, or to perform
integration tests, you will need to have a server running with the correct
configuration. To do this, follow these steps:
1. Have the docker infrastructure running: `docker compose up`
Before running against a live API (`yarn test-integration` and `yarn
test-record`), you need to boot up the infrastructure and ensure that the
nomad package is available with the correct test configuration:
2. Have the `nomad appworker` running with the config found in
`gui/tests/nomad.yaml`. This can be achieved e.g. with the command: `export
NOMAD_CONFIG=gui/tests/nomad.yaml; nomad admin run appworker`
1. Have the docker infrastructure running: `docker compose up`
3. Activate the correct python virtual environment before running the tests
with yarn (yarn will run the python functions that prepare the state).
2. Have the `nomad appworker` running with the config found in
`gui/tests/nomad.yaml`. This can be achieved e.g. with the command: `export
NOMAD_CONFIG=gui/tests/nomad.yaml; nomad admin run appworker`
4. Run the tests with `yarn test-record [<filename>]` if you wish to record a
snapshot file or `yarn test-integration [<filename>]` if you want the
perform the test without any recording.
3. Activate the correct python virtual environment before running the tests
with yarn (yarn will run the python functions that prepare the state).
## Build the docker image
......
Plugins allow you to add Python-based functionality to NOMAD without a custom build
NOMAD image or release. Plugins can be installed at NOMAD start-up time. Therefore, you can
configure each NOMAD (Oasis) with a different custom set of plugins or disable unnecessary
plugins.
We support different kinds of plugins:
- Python **schema**, read also [Python schema documentation](schema/python.md).
- **parser**, read also [parser development documentation](develop/parser.md).
- **normalizer** (coming soon...)
- additional custom **APIs** (coming soon...)
## Develop a plugin
We provide template projects on GitHub. You can fork these projects and follow the
instructions in their `README.md`. These instructions will give you everything you
need to run and test your plugin as a plugin developer.
The following sections here contain more background information and explain how to
add plugins to a NOMAD installation.
- [schema plugin](https://github.com/nomad-coe/nomad-schema-plugin-example)
- [parser plugin](https://github.com/nomad-coe/nomad-parser-plugin-example)
### Plugin anatomy
A plugin usually consist of the *plugin code* (a Python package) and
*plugin metadata*. The installation independent *plugin metadata* (e.g. name, description, python package, etc.)
can be defined in a `nomad_plugin.yaml` that is part of the *plugin code*.
The installation dependent *plugin metadata* (e.g. plugin key, order and priority, parser matching rules, etc.)
is added to the `nomad.yaml` of the NOMAD installation.
Here is the project layout of the schema example:
```
my-nomad-schema
├── nomadschemaexample
│ ├── __init__.py
│ ├── nomad_plugin.yaml
│ └── schema.py
├── tests
│ ├── data
│ │ └── test.archive.yaml
│ └── test_schema.py
├── LICENSE
├── README.md
├── nomad.yaml
└── requirements.txt
```
### Plugin code
The directory `nomadschemaexample` is our Python package *plugin code*. In this case,
it contains a simple `schema.py`. Read the [Python schema documentation](schema/python.md)
for more details:
```python
{{ file_contents('examples/plugins/schema/nomadschemaexample/schema.py') }}
```
### Plugin metadata
The file `nomad_plugin.yaml` contains the installation independent *plugin metadata*:
```yaml
{{ file_contents('examples/plugins/schema/nomadschemaexample/nomad_plugin.yaml') }}
```
The metadata contains the `plugin_type` (e.g. `schema` or `parser`). The rest of the
yaml will depend on the type and the underlying metadata model. For schemas there are only
descriptive metadata like `name` or `description` as schemas do not contain any technical
metadata that is necessary to use them. See below for a reference of the *plugin metadata*
models.
The file `nomad.yaml` shows how to add the plugin to a nomad installation. As a plugin
developer you have [installed our Python package](./pythonlib.md) and can run the `nomad parse`
command as your "installation" to try your schema:
```yaml
{{ file_contents('examples/plugins/schema/nomad.yaml') }}
```
Plugins are defined under the `plugins` key. This consists of `include` (or `exclude`) to
select a subset of all plugins defined under `options`. The `options` given in the
`nomad.yaml` will be merged with all the default "plugins" that come with NOMAD (mostly the NOMAD parsers at the moment).
In this example, we disable all default plugins by just including our `schemas/example`.
The `options` field can be used to add define the *code* and installation independent *metadata*
via the `python_package` key. But you can overwrite or add more *metadata* keys here as well:
each `options` entry with `python_package` will be merged with the data in the package's
`nomad_plugin.yaml`.
Please note that `python_package` is the name of a Python package and not a path to the
code. This also means that the package has to be in your `PYTHONPATH` (see below).
{{pydantic_model('nomad.config.plugins.Schema', heading='### Parser plugin metadata')}}
{{pydantic_model('nomad.config.plugins.Parser', heading='### Schema plugin metadata')}}
Now follow the instructions for one of our examples and try for yourself:
- [schema plugin](https://github.com/nomad-coe/nomad-schema-plugin-example)
- [parser plugin](https://github.com/nomad-coe/nomad-parser-plugin-example)
## Add a plugin to your NOMAD
To add a plugin, you need to add the *plugin metadata* to `nomad.yaml` (see above) and you need
to add the *plugin code* to the `PYTHONPATH` of your NOMAD. The `nomad.yaml` needs to be
edited manually in the usual way. There are several ways to
add *plugin code* to a NOMAD installation.
### Development setup of NOMAD
Simply add the plugin directory to the `PYTHONPATH` environment variable. When you start
the application (e.g. `nomad admin run appworker`), Python will find your code when NOMAD
imports the `python_package` given in the `plugins.options` of your `nomad.yaml`.
### NOMAD Oasis
The NOMAD docker image adds the folder `/app/plugins` to the `PYTHONPATH`. You simply have
to add the *plugin metadata* to your Oasis' `nomad.yaml` and mount your code into the `/app/plugins`
directory via the volumes section of the `app` and `worker` services in your `docker-compose.yaml`.
For example, you can do this by adding an extension to the `docker-compose.yaml`, e.g. a file called
`docker-compose.plugins.yaml`. Assuming you cloned the example plugins above into the Oasis folder as
`./nomad-schema-plugin-example` and `./nomad-parser-plugin-example`,
your `docker-compose.plugins.yaml` should look like this:
```yaml
services:
worker:
volumes:
- ./nomad-schema-plugin-example/nomadschemaexample:/app/plugins/nomadschemaexample
- ./nomad-parser-plugin-example/nomadparserexample:/app/plugins/nomadparserexample
app:
volumes:
- ./nomad-schema-plugin-example/nomadschemaexample:/app/plugins/nomadschemaexample
- ./nomad-parser-plugin-example/nomadparserexample:/app/plugins/nomadparserexample
```
You have to tell docker that there are now two compose files. This can be done via the
`COMPOSE_FILE` environment variable. This is how you can start the Oasis with the plugins:
```sh
export COMPOSE_FILE=docker-compose.yaml:docker-compose.plugins.yaml
docker compose up -d
```
Here is a complete Oasis setup [nomad-oasis-with-plugins.zip](assets/nomad-oasis-with-plugins.zip).
Simply download, extract, and start like any other Oasis:
```sh
unzip nomad-oasis-with-plugins.zip
cd nomad-oasis-with-plugins
sudo chown -R 1000 .volumes
export COMPOSE_FILE=docker-compose.yaml:docker-compose.plugins.yaml
docker compose pull
docker compose up -d
curl localhost/nomad-oasis/alive
```
Read the [Oasis documentation](oasis.md) for more details.
### Other means
- via python packages (coming soon...)
- via github projects (coming soon...)
## Publish a plugin
coming soon...
We plan to provide a plugin registry that allows you to publish your plugin's *metadata*.
This can then be used to simplify plugin management within a NOMAD installation.
\ No newline at end of file
Subproject commit 5059c528828875b94210f7130eed0670372ae46f
Subproject commit b6642e43b768a01aa5ded08af56980e144d204c1
REACT_APP_BACKEND_URL=http://localhost:8000/fairdi/nomad/latest
......@@ -64,6 +64,7 @@
"recoil": "0.7.5",
"recompose": "^0.30.0",
"remark": "^14.0.2",
"remark-gfm": "^3.0.1",
"remark-math": "^5.1.1",
"swagger-client": "^3.8.22",
"tinymce": "^6.0.3",
......
This diff is collapsed.