Commit 215a1934 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch 'parser-move' into 'v0.10.0'

Parser move

See merge request !280
parents d8984d35 373fd704
Pipeline #95607 passed with stages
in 22 minutes and 48 seconds
...@@ -34,4 +34,6 @@ parser.osio.log ...@@ -34,4 +34,6 @@ parser.osio.log
gui/src/metainfo.json gui/src/metainfo.json
gui/src/searchQuantities.json gui/src/searchQuantities.json
examples/workdir/ examples/workdir/
gunicorn.log.conf
nomad/gitinfo.py
*/node_modules/ */node_modules/
...@@ -147,6 +147,8 @@ dev setup tests: ...@@ -147,6 +147,8 @@ dev setup tests:
- pip install --upgrade pip - pip install --upgrade pip
- ./setup.sh - ./setup.sh
- python -m nomad.cli parse tests/data/parsers/vasp/vasp.xml - python -m nomad.cli parse tests/data/parsers/vasp/vasp.xml
- python -c "import nomad.app"
- python -c "import nomad.processing"
except: except:
refs: refs:
- tags - tags
......
...@@ -4,24 +4,24 @@ ...@@ -4,24 +4,24 @@
branch = nomad-fair branch = nomad-fair
[submodule "dependencies/parsers/vasp"] [submodule "dependencies/parsers/vasp"]
path = dependencies/parsers/vasp path = dependencies/parsers/vasp
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-vasp.git url = https://github.com/nomad-coe/nomad-parser-vasp.git
branch = nomad-fair branch = master
[submodule "dependencies/parsers/exciting"] [submodule "dependencies/parsers/exciting"]
path = dependencies/parsers/exciting path = dependencies/parsers/exciting
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-exciting-fairdi.git url = https://github.com/nomad-coe/nomad-parser-exciting.git
branch = nomad-fair branch = master
[submodule "dependencies/parsers/fhi-aims"] [submodule "dependencies/parsers/fhi-aims"]
path = dependencies/parsers/fhi-aims path = dependencies/parsers/fhi-aims
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-fhi-aims.git url = https://github.com/nomad-coe/nomad-parser-fhi-aims.git
branch = nomad-fair branch = master
[submodule "dependencies/parsers/cp2k"] [submodule "dependencies/parsers/cp2k"]
path = dependencies/parsers/cp2k path = dependencies/parsers/cp2k
url = https://github.com/nomad-coe/nomad-parser-cp2k.git url = https://github.com/nomad-coe/nomad-parser-cp2k.git
branch = master branch = master
[submodule "dependencies/parsers/crystal"] [submodule "dependencies/parsers/crystal"]
path = dependencies/parsers/crystal path = dependencies/parsers/crystal
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-crystal.git url = https://github.com/nomad-coe/nomad-parser-crystal.git
branch = nomad-fair branch = master
[submodule "dependencies/parsers/cpmd"] [submodule "dependencies/parsers/cpmd"]
path = dependencies/parsers/cpmd path = dependencies/parsers/cpmd
url = https://github.com/nomad-coe/nomad-parser-cpmd.git url = https://github.com/nomad-coe/nomad-parser-cpmd.git
...@@ -52,8 +52,8 @@ ...@@ -52,8 +52,8 @@
branch = nomad-fair branch = nomad-fair
[submodule "dependencies/parsers/abinit"] [submodule "dependencies/parsers/abinit"]
path = dependencies/parsers/abinit path = dependencies/parsers/abinit
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-abinit.git url = https://github.com/nomad-coe/nomad-parser-abinit.git
branch = nomad-fair branch = master
[submodule "dependencies/parsers/orca"] [submodule "dependencies/parsers/orca"]
path = dependencies/parsers/orca path = dependencies/parsers/orca
url = https://github.com/nomad-coe/nomad-parser-orca.git url = https://github.com/nomad-coe/nomad-parser-orca.git
...@@ -100,8 +100,8 @@ ...@@ -100,8 +100,8 @@
branch = master branch = master
[submodule "dependencies/parsers/elastic"] [submodule "dependencies/parsers/elastic"]
path = dependencies/parsers/elastic path = dependencies/parsers/elastic
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-elastic.git url = https://github.com/nomad-coe/nomad-parser-elastic.git
branch = nomad-fair branch = master
[submodule "dependencies/parsers/gamess"] [submodule "dependencies/parsers/gamess"]
path = dependencies/parsers/gamess path = dependencies/parsers/gamess
url = https://github.com/nomad-coe/nomad-parser-gamess.git url = https://github.com/nomad-coe/nomad-parser-gamess.git
...@@ -175,14 +175,16 @@ ...@@ -175,14 +175,16 @@
branch = master branch = master
[submodule "dependencies/parsers/gromacs"] [submodule "dependencies/parsers/gromacs"]
path = dependencies/parsers/gromacs path = dependencies/parsers/gromacs
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-gromacs.git url = https://github.com/nomad-coe/nomad-parser-gromacs.git
branch = master
[submodule "dependencies/parsers/gromos"] [submodule "dependencies/parsers/gromos"]
path = dependencies/parsers/gromos path = dependencies/parsers/gromos
url = https://github.com/nomad-coe/nomad-parser-gromos.git url = https://github.com/nomad-coe/nomad-parser-gromos.git
branch = master branch = master
[submodule "dependencies/parsers/lammps"] [submodule "dependencies/parsers/lammps"]
path = dependencies/parsers/lammps path = dependencies/parsers/lammps
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-lammps.git url = https://github.com/nomad-coe/nomad-parser-lammps.git
branch = master
[submodule "dependencies/parsers/openkim"] [submodule "dependencies/parsers/openkim"]
path = dependencies/parsers/openkim path = dependencies/parsers/openkim
url = https://github.com/nomad-coe/nomad-parser-openkim.git url = https://github.com/nomad-coe/nomad-parser-openkim.git
......
Subproject commit 562ae7d26fa108c7c31ebe38775c767aba1e642c Subproject commit f55b5ad7826793c20d93a73fab01d028fbb44b00
Subproject commit e2c97b91a97ff997b41a99df9595ba6faea15f36 Subproject commit b6b9715460a414af4af433bc44a832c97319080f
Subproject commit d4a5d39b6c012807d71597a3e695295e51b4eedd Subproject commit 6dfe32362b392808399fb9eba1853d522b8956bf
Subproject commit 7533d541798b110f17873f640b726cf9d7c6302e Subproject commit d43d7cb50682679728f7280be309976feb4353be
Subproject commit 9246d3e496073282aa56b920e6d075a97829ffce Subproject commit 44d5861d0503976714ce1a1e09bac2aa3495ddf9
Subproject commit f9870d4dfb52c2a89a90ef1497cef45eac59a8f7 Subproject commit 139ab4a5e288542f4343db3419597fdeddbde3f4
Subproject commit a32a1499b6aa163b17fe01fc8d324b55779418ec Subproject commit bf2182de246d0b63866e8a61b94392afb8b78073
Subproject commit 7a9433b319a9a01445fc4542e3ba1f5258c5698c Subproject commit 6ff8642f195b4eab1f6284d51f455348763602b6
Subproject commit c09f64143b6df098bd06882465fb9e4d29c44578 Subproject commit ce55910cabb529b7c0751017c5f10e803e2594a3
Subproject commit 4c35bae7f2834e9657eea438c0a98f8810ac415b Subproject commit 6ebe206cb99f5bdf008f81f7bc8e4e434414460d
...@@ -76,6 +76,10 @@ everything into your virtual environment: ...@@ -76,6 +76,10 @@ everything into your virtual environment:
./dependencies.sh -e ./dependencies.sh -e
``` ```
If one of the Python packages that are installed during this process, fails because it
cannot be compiled on your platform, you can try `pip install --prefer-binary <packagename>`
to install set package manually.
The `-e` option will install the NOMAD-coe dependencies with symbolic links allowing you The `-e` option will install the NOMAD-coe dependencies with symbolic links allowing you
to change the downloaded dependency code without having to reinstall after. to change the downloaded dependency code without having to reinstall after.
...@@ -134,11 +138,20 @@ via your preferred tools. Just make sure to use the right ports. ...@@ -134,11 +138,20 @@ via your preferred tools. Just make sure to use the right ports.
## Running NOMAD ## Running NOMAD
NOMAD consist of the NOMAD app/api, a worker, and the GUI. You can run app and worker with NOMAD consist of the NOMAD app/api, a worker, and the GUI. You can run app and worker with
the NOMAD cli: the NOMAD cli. These commands will run the services and show their logout put. You should open
them in separate shells as they run continuously. They will not watch code changes and
you have to restart manually.
```sh ```sh
nomad admin run app nomad admin run app
```
```sh
nomad admin run worker nomad admin run worker
```
Or both together in once process:
```
nomad admin run appworker nomad admin run appworker
``` ```
...@@ -150,7 +163,9 @@ celery -A nomad.processing worker -l info ...@@ -150,7 +163,9 @@ celery -A nomad.processing worker -l info
``` ```
When you run the gui on its own (e.g. with react dev server below), you have to have When you run the gui on its own (e.g. with react dev server below), you have to have
the app manually also. the app manually also. The gui and its dependencies run on [node](https://nodejs.org) and
the [yarn](https://yarnpkg.com/) dependency manager. Read their documentation on how to
install them for your platform.
```sh ```sh
cd gui cd gui
yarn yarn
...@@ -162,18 +177,11 @@ yarn start ...@@ -162,18 +177,11 @@ yarn start
To run the tests some additional settings and files are necessary that are not part To run the tests some additional settings and files are necessary that are not part
of the code base. of the code base.
First you need to create a `nomad.yaml` with the admin password for the user management First, you need to provide the `springer.msg` Springer materials database. It can
system:
```yaml
keycloak:
password: <the-password>
```
Secondly, you need to provide the `springer.msg` Springer materials database. It can
be copied from `/nomad/fairdi/db/data/springer.msg` on our servers and should be copied from `/nomad/fairdi/db/data/springer.msg` on our servers and should
be placed at `nomad/normalizing/data/springer.msg`. be placed at `nomad/normalizing/data/springer.msg`.
Thirdly, you have to provide static files to serve the docs and NOMAD distribution: Second, you have to provide static files to serve the docs and NOMAD distribution:
```sh ```sh
cd docs cd docs
make html make html
......
...@@ -66,7 +66,7 @@ def wrap_positions( ...@@ -66,7 +66,7 @@ def wrap_positions(
cell: NDArray[Any] = None, cell: NDArray[Any] = None,
pbc: Union[bool, NDArray[Any]] = True, pbc: Union[bool, NDArray[Any]] = True,
center: NDArray[Any] = [0.5, 0.5, 0.5], center: NDArray[Any] = [0.5, 0.5, 0.5],
eps: float = 1e-7) -> NDArray[Any]: eps: float = 1e-12) -> NDArray[Any]:
"""Wraps the given position so that they are within the unit cell. If no """Wraps the given position so that they are within the unit cell. If no
cell is given, scaled positions are assumed. For wrapping cartesian cell is given, scaled positions are assumed. For wrapping cartesian
positions you also need to provide the cell. positions you also need to provide the cell.
...@@ -94,13 +94,14 @@ def wrap_positions( ...@@ -94,13 +94,14 @@ def wrap_positions(
if cell is None: if cell is None:
fractional = positions fractional = positions
else: else:
fractional = to_scaled(positions) - shift fractional = to_scaled(positions, cell)
fractional -= shift
for i, periodic in enumerate(pbc): for i, periodic in enumerate(pbc):
if periodic: if periodic:
fractional[:, i] %= 1.0 fractional[:, i] %= 1.0
fractional[:, i] += shift[i] fractional[:, i] += shift[i]
if cell: if cell is not None:
return np.dot(fractional, cell) return np.dot(fractional, cell)
else: else:
return fractional return fractional
......
...@@ -69,6 +69,10 @@ def __run_parallel( ...@@ -69,6 +69,10 @@ def __run_parallel(
cv.notify() cv.notify()
for upload in uploads: for upload in uploads:
logger.info(
'cli schedules parallel %s processing for upload' % label,
current_process=upload.current_process,
current_task=upload.current_task, upload_id=upload.upload_id)
with cv: with cv:
cv.wait_for(lambda: state['available_threads_count'] > 0) cv.wait_for(lambda: state['available_threads_count'] > 0)
state['available_threads_count'] -= 1 state['available_threads_count'] -= 1
...@@ -84,6 +88,10 @@ def __run_processing( ...@@ -84,6 +88,10 @@ def __run_processing(
uploads, parallel: int, process, label: str, reprocess_running: bool = False): uploads, parallel: int, process, label: str, reprocess_running: bool = False):
def run_process(upload, logger): def run_process(upload, logger):
logger.info(
'cli calls %s processing' % label,
current_process=upload.current_process,
current_task=upload.current_task, upload_id=upload.upload_id)
if upload.process_running and not reprocess_running: if upload.process_running and not reprocess_running:
logger.warn( logger.warn(
'cannot trigger %s, since the upload is already/still processing' % label, 'cannot trigger %s, since the upload is already/still processing' % label,
......
...@@ -99,7 +99,7 @@ def rabbitmq_url(): ...@@ -99,7 +99,7 @@ def rabbitmq_url():
celery = NomadConfig( celery = NomadConfig(
max_memory=64e6, # 64 GB max_memory=64e6, # 64 GB
timeout=1800, # 1/2 h timeout=1800, # 1/2 h
acks_late=True, acks_late=False,
routing=CELERY_QUEUE_ROUTING, routing=CELERY_QUEUE_ROUTING,
priorities={ priorities={
'Upload.process_upload': 5, 'Upload.process_upload': 5,
...@@ -288,6 +288,7 @@ meta = NomadConfig( ...@@ -288,6 +288,7 @@ meta = NomadConfig(
version='0.10.0', version='0.10.0',
commit=gitinfo.commit, commit=gitinfo.commit,
release='devel', release='devel',
deployment='standard',
default_domain='dft', default_domain='dft',
service='unknown nomad service', service='unknown nomad service',
name='novel materials discovery (NOMAD)', name='novel materials discovery (NOMAD)',
...@@ -310,6 +311,8 @@ raw_file_strip_cutoff = 1000 ...@@ -310,6 +311,8 @@ raw_file_strip_cutoff = 1000
max_entry_download = 500000 max_entry_download = 500000
use_empty_parsers = False use_empty_parsers = False
reprocess_unmatched = True reprocess_unmatched = True
reprocess_rematch = True
process_reuse_parser = True
metadata_file_name = 'nomad' metadata_file_name = 'nomad'
metadata_file_extensions = ('json', 'yaml', 'yml') metadata_file_extensions = ('json', 'yaml', 'yml')
......
...@@ -415,6 +415,10 @@ class EntryMetadata(metainfo.MSection): ...@@ -415,6 +415,10 @@ class EntryMetadata(metainfo.MSection):
categories=[MongoMetadata], categories=[MongoMetadata],
a_search=Search()) a_search=Search())
processing_errors = metainfo.Quantity(
type=str, shape=['*'], description='Errors that occured during processing',
a_search=Search(many_and='append'))
nomad_version = metainfo.Quantity( nomad_version = metainfo.Quantity(
type=str, type=str,
description='The NOMAD version used for the last processing', description='The NOMAD version used for the last processing',
......
...@@ -1523,7 +1523,6 @@ class Dos(MSection): ...@@ -1523,7 +1523,6 @@ class Dos(MSection):
dos_values_lm = Quantity( dos_values_lm = Quantity(
type=np.dtype(np.float64), type=np.dtype(np.float64),
shape=['number_of_dos_lms', 'number_of_spin_channels', 'number_of_atoms', 'number_of_dos_values'], shape=['number_of_dos_lms', 'number_of_spin_channels', 'number_of_atoms', 'number_of_dos_values'],
unit='joule',
description=''' description='''
Array containing the density (electronic-energy) of states values projected on the Array containing the density (electronic-energy) of states values projected on the
various spherical harmonics (integrated on all atoms), see various spherical harmonics (integrated on all atoms), see
......
log, ref, version, commit = "44c7e69 Fixed performance and other isseus.", "heads/mui4", "v0.7.9-398-g44c7e69", "44c7e69"
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment