Commit 873efcae authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch 'v0.10.0' into 'master'

Merge v0.10.0 into master for release

Closes #475, #484, #497, #492, #498, and #500

See merge request !283
parents 5636e115 015742db
Pipeline #95934 passed with stage
in 2 minutes and 28 seconds
Subproject commit afdd0231addcc93555ac371ff7ae59b03b9ba6e2
Subproject commit 8e6edb5af2fd5044e72780d21e77d0a6e3038804
Subproject commit a3b4c120bd5eff3d108eca891fe7ef6109f3a964
Subproject commit b8c77180798d399cbf5e1b242187ce8230b041c3
Subproject commit 76c3ef2e319b704e6dd64b9ea670d6748b89d2c1
Subproject commit 02c12ef1f72f23956febc463e69042e7902d2765
Subproject commit 86720f57c32a3bb36eb2896b56d7628745b18abb
Subproject commit 382ff8607e3f0a4e2535fe68bccae9e0d0e3ed0c
Subproject commit 234b92ed8ef8b7e0a2e95d8255f841d4531c24ed
Subproject commit bd9cd9307d0598f4c9a7bc57f0f469d1f9e8fa34
Subproject commit 2dba3f3531a96a293e8d62db9c05949ff7e591c1
Subproject commit 400ccf334212382aac2b1b424e5834dc979cf5af
Subproject commit a6f432f5356db5ea5f29b9a401f007dbb539ac8c
Subproject commit 342c83c8a935d02ff74e4479df05eac5562cf237
Subproject commit 4c9ffd8c1d6bb99ffa263b279807d979040fbc50
Subproject commit 1cd31deb8c043df3f72067b5b566e19796cec2aa
Subproject commit fae4d10b3ee0513d269a00f965479b8aeae63de0
Subproject commit 271105a96fb0d6a1689cf11042db5ef3d01e7d08
Subproject commit 3a846398a4727ce98f1bbad9c59b1aae6e07af74
Subproject commit ce55910cabb529b7c0751017c5f10e803e2594a3
Subproject commit 163e232f3d24ead464bb8f8baf893087fe8c18da
Subproject commit 55f1c7ad339e6b8bf16d1020ceb9d46cd106c029
Subproject commit aac537c894f9d7b6caeb19d68c8e23e67c0706c7
Subproject commit c9e2bd0c179e85d881594c68cc2c0d53f3e59bf4
Subproject commit 2775787b5b0c1a0d2f4bf21f5903cddb99aa81e0
Subproject commit eae5d87390863b72875087b1e69cfe15a516a5b9
Subproject commit 6faaa3508f0db2d35f103cffd2c003459a93c82e
Subproject commit e51352236af14d8f1a4345f8570329f00bec1b5e
Subproject commit 56eb84fffa3ae25a4b8dc10aa56da85a4fe6584d
Subproject commit 6ebe206cb99f5bdf008f81f7bc8e4e434414460d
Subproject commit 0bfb17daec156d26110d55cd0918d8fb584f7fac
Subproject commit c140f8c0765f4c5c1dbefe6b8c9803ea17018ede
Subproject commit c1aca04237d69097bbeb17d3a397be66e9c6797f
Subproject commit 725217b401a130b34260f29334e353c9548cd1fa
......@@ -11,7 +11,7 @@ consult our *swagger* dashboards:
Summary
-------
.. qrefflask:: nomad.app:app
.. qrefflask:: nomad.app.flask:app
:undoc-static:
......@@ -19,5 +19,5 @@ Summary
API(s) Details
--------------
.. autoflask:: nomad.app:app
.. autoflask:: nomad.app.flask:app
:undoc-static:
......@@ -76,6 +76,10 @@ everything into your virtual environment:
./dependencies.sh -e
```
If one of the Python packages that are installed during this process, fails because it
cannot be compiled on your platform, you can try `pip install --prefer-binary <packagename>`
to install set package manually.
The `-e` option will install the NOMAD-coe dependencies with symbolic links allowing you
to change the downloaded dependency code without having to reinstall after.
......@@ -134,11 +138,20 @@ via your preferred tools. Just make sure to use the right ports.
## Running NOMAD
NOMAD consist of the NOMAD app/api, a worker, and the GUI. You can run app and worker with
the NOMAD cli:
the NOMAD cli. These commands will run the services and show their logout put. You should open
them in separate shells as they run continuously. They will not watch code changes and
you have to restart manually.
```sh
nomad admin run app
```
```sh
nomad admin run worker
```
Or both together in once process:
```
nomad admin run appworker
```
......@@ -150,7 +163,9 @@ celery -A nomad.processing worker -l info
```
When you run the gui on its own (e.g. with react dev server below), you have to have
the app manually also.
the app manually also. The gui and its dependencies run on [node](https://nodejs.org) and
the [yarn](https://yarnpkg.com/) dependency manager. Read their documentation on how to
install them for your platform.
```sh
cd gui
yarn
......@@ -213,7 +228,6 @@ line size ruler, etc.
{
"python.venvPath": "${workspaceFolder}/.pyenv",
"python.pythonPath": "${workspaceFolder}/.pyenv/bin/python",
"git.ignoreLimitWarning": true,
"editor.rulers": [90],
"editor.renderWhitespace": "all",
"editor.tabSize": 4,
......@@ -221,29 +235,20 @@ line size ruler, etc.
"editor.tabSize": 2
},
"files.trimTrailingWhitespace": true,
"git.enableSmartCommit": true,
"eslint.autoFixOnSave": true,
"editor.codeActionsOnSave": ["source.fixAll.eslint"],
"python.linting.pylintEnabled": true,
"python.linting.pylintArgs": [
"--load-plugins=pylint_mongoengine,nomad/metainfo/pylint_plugin",
],
"python.linting.pep8Path": "pycodestyle",
"python.linting.pep8Enabled": true,
"python.linting.pep8Args": ["--ignore=E501,E701"],
"python.linting.pycodestylePath": "pycodestyle",
"python.linting.pycodestyleEnabled": true,
"python.linting.pycodestyleArgs": ["--ignore=E501,E701,E731"],
"python.linting.mypyEnabled": true,
"python.linting.mypyArgs": [
"--ignore-missing-imports",
"--follow-imports=silent",
"--no-strict-optional"
],
"workbench.colorCustomizations": {
"editorError.foreground": "#FF2222",
"editorOverviewRuler.errorForeground": "#FF2222",
"editorWarning.foreground": "#FF5500",
"editorOverviewRuler.warningForeground": "#FF5500",
"activityBar.background": "#4D2111",
"titleBar.activeBackground": "#6B2E18",
"titleBar.activeForeground": "#FDF9F7"
},
"files.watcherExclude": {
"**/.git/objects/**": true,
"**/.git/subtree-cache/**": true,
......
'''
The scenario is that you have a lot of <name>-*.zip files for upload. This script
will add a nomad.json to the uploads, upload them 1 at a time, watch the processing, repeat.
'''
from typing import Dict, Any
from bravado.requests_client import RequestsClient, Authenticator
from bravado.client import SwaggerClient
from keycloak import KeycloakOpenID
from urllib.parse import urlparse
import time
import os.path
import sys
import zipfile
from nomad.client import KeycloakAuthenticator
from nomad import config
nomad_url = 'http://nomad-lab.eu/prod/rae/api'
user = 'youruser'
password = 'yourpassword'
uploader_id = None
# create the bravado client
http_client = RequestsClient()
http_client.authenticator = KeycloakAuthenticator(
host=urlparse(nomad_url).netloc,
user=user,
password=password,
server_url=config.keycloak.server_url,
realm_name=config.keycloak.realm_name,
client_id=config.keycloak.client_id)
client = SwaggerClient.from_url('%s/swagger.json' % nomad_url, http_client=http_client)
def upload(
path: str, local_path: bool = False, metadata_path: str = None,
publish_directly: bool = False, uploader_id: str = None):
'''
Arguments:
path: The file path to the upload file.
local: If this file should be uploaded with &local_path
metadata_path: Optional nomad.(yaml|json) metadata file
publish_directly: If the upload should be published directly
'''
assert os.path.isfile(path), f'The {path} is not a file'
# add metadata
if metadata_path is not None:
assert os.path.isfile(metadata_path), f'The {metadata_path} is not a file'
assert os.path.basename(metadata_path).endswith('.json'), f'The {metadata_path} is not a nomad metadata file'
assert path.endswith('.zip'), 'Adding nomad metadata is only supported for .zip files'
with zipfile.ZipFile(path, 'a') as zip:
zip.write(metadata_path, 'nomad.json')
# upload
print(f'uploading {path}')
kwargs: Dict[str, Any] = {}
if publish_directly:
kwargs['publish_directly'] = True
if uploader_id is not None:
kwargs['uploader_id'] = uploader_id
if local_path:
upload = client.uploads.upload(local_path=path, **kwargs).response().result
else:
with open(path, 'rb') as f:
upload = client.uploads.upload(file=f, **kwargs).response().result
print(f'processing {path}')
while upload.tasks_running:
upload = client.uploads.get_upload(upload_id=upload.upload_id).response().result
time.sleep(3)
print('processed: %d, failures: %d' % (upload.processed_calcs, upload.failed_calcs))
# check if processing was a success
if upload.tasks_status != 'SUCCESS':
print('something went wrong')
print('errors: %s' % str(upload.errors))
# try to delete the unsuccessful upload
client.uploads.delete_upload(upload_id=upload.upload_id).response().result
return False
return True
if __name__ == '__main__':
metadata_path = None
if sys.argv[1].endswith('json'):
metadata_path = sys.argv[1]
paths = sys.argv[2:]
else:
paths = sys.argv[1:]
for path in paths:
upload(
path, metadata_path=metadata_path, local_path=True, publish_directly=True,
uploader_id=uploader_id)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment