Commit ec46f648 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch 'migration' into 'master'

Merge version 0.4.0

See merge request !33
parents e6fb7d44 9d6dd97b
Pipeline #44578 canceled with stages
in 14 minutes and 21 seconds
......@@ -72,7 +72,9 @@ linting:
- python -m pylint --load-plugins=pylint_mongoengine nomad tests
- python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests
except:
- /^dev-.*$/
variables:
- $CI_COMMIT_REF_NAME =~ /^dev-.*$/
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
tests:
stage: test
......@@ -104,7 +106,9 @@ tests:
- cd /app
- python -m pytest --cov=nomad -sv tests
except:
- /^dev-.*$/
variables:
- $CI_COMMIT_REF_NAME =~ /^dev-.*$/
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
# does currently not work, current GitLab CI runner does not network services with each other
# integration-tests:
......@@ -207,9 +211,12 @@ deploy:
- export FILES_PATH="/scratch/nomad-fair/fs/nomad_v${NOMAD_VERSION}"
- if [ ${DEPLOYS} -eq 0 ]; then
helm install --name=${RELEASE_NAME} . --namespace=${STAGING_NAMESPACE}
--set api.disableReset="false"
--set proxy.nodePort="300${NUMERIC_VERSION//./}"
--set proxy.external.path=${EXTERNAL_PATH}
--set dbname=${DBNAME}
--set worker.replicas=1
--set worker.memrequest=32
--set volumes.files=${FILES_PATH};
else
helm upgrade ${RELEASE_NAME} . --namespace=${STAGING_NAMESPACE} --recreate-pods;
......
......@@ -51,3 +51,22 @@
[submodule "dependencies/parsers/quantum-espresso"]
path = dependencies/parsers/quantum-espresso
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-quantum-espresso
[submodule "dependencies/parsers/abinit"]
path = dependencies/parsers/abinit
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-abinit
[submodule "dependencies/parsers/orca"]
path = dependencies/parsers/orca
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-orca
[submodule "dependencies/parsers/castep"]
path = dependencies/parsers/castep
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-castep
[submodule "dependencies/parsers/dl-poly"]
path = dependencies/parsers/dl-poly
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-dl-poly
[submodule "dependencies/parsers/lib-atoms"]
path = dependencies/parsers/lib-atoms
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-lib-atoms
[submodule "dependencies/parsers/octopus"]
path = dependencies/parsers/octopus
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-octopus
......@@ -138,16 +138,65 @@
]
},
{
"name": "Python: Quantum Espresso",
"name": "Quantum Espresso Normalizer",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_parsing.py::test_parser[parsers/quantumespresso-tests/data/parsers/quantum-espresso/W.out]"
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/quantumespresso-tests/data/parsers/quantum-espresso/W.out]"
]
},
{
"name": "Abinit Normalizer",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/abinit-tests/data/parsers/abinit/Fe.out]"
]
},
{
"name": "Castep Normalizer",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/castep-tests/data/parsers/castep/BC2N-Pmm2-Raman.castep]"
]
},
{
"name": "DL-Poly Normalizer",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/dl-poly-tests/data/parsers/dl-poly/OUTPUT]"
]
},
{
"name": "Lib Atoms Normalizer",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/lib-atoms-tests/data/parsers/lib-atoms/gp.xml]"
]
},
{
"name": "Octopus Normalizer",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer[parsers/octopus-tests/data/parsers/octopus/stdout.txt]"
]
},
{
"name": "Python: Current File",
"type": "python",
......
Subproject commit 8957235e5ca6bbb3278790e2587a30323169610f
Subproject commit b232a5fdd11c344656ff1e6ec7a2ed6f787cf49f
Subproject commit 18f8ccde48f39a9237882064e20e5a1e7baff5c0
Subproject commit b68bce7db5c02ac56b1cf892ae20458fffb23b65
Subproject commit 71f7a2ad0d77d376e30d3c1a8ae55920fc7d7e5c
Subproject commit 5c7cf8add659c51d940960daa430927923720f7e
Subproject commit de6d49f89748010236baa9b683464b70bc8507a6
Subproject commit d7bfc62deebfd40d3033817c529d1f6f89c901e5
Subproject commit 7de9f91273f4c5becaaad7fef8710e68b3cd63de
Subproject commit 83a4de4efa40731f7e192c795fc1e82f00f27fc8
Subproject commit 2a59119a6635454eba33c268d8cf8caa6eecd204
Subproject commit fcda3a9ab4a23ea0c7363955999943d9be6fa0fb
# API Tutorial
This tutorial assumes that you want to
- upload some data
- publish the data
- find it
- download it again
## Prequisites
### Python
The tutorial was tested with Python 3, but it might as well work with Python 2.
### Python packages
We do not assume many specific python packages. Only the *bravado* package (available
via pipy) is required. It allows us to use the nomad ReST API in a more friendly and
pythonic way. You can simply install it the usual way
```
pip install bravado
```
For the following code snippets, we need the following imports:
```python
from bravado.requests_client import RequestsClient
from bravado.client import SwaggerClient
from bravado.exception import HTTPNotFound
from urllib.parse import urlparse
import time
import os.path
import sys
```
### An example file
Lets assume you have an example upload file ready. Its a `.zip` (`.tgz` would also work)
with some *VASP* data from a single run at `/example/AcAg/vasprun.xml`, `/example/AcAg/OUTCAR`, ...
Lets keep the filename in a variable:
```python
upload_file = 'example.zip'
```
### Nomad
We need to know the nomad installation to use and its respective API URL. To upload
data you also need an account (email, password):
```python
nomad_url = 'http://enc-staging-nomad.esc.rzg.mpg.de/fairdi/nomad/v0.3.0/api'
user = 'leonard.hofstadter@nomad-fairdi.tests.de'
password = 'password'
```
### Using bravado
Bravado reads a ReST API's definition from a `swagger.json` as it is provided by
many APIs, including nomad's of course. Bravado also allows to use authentication,
which makes it even easier. The following would be a typical setup:
```python
host = urlparse(nomad_url).netloc.split(':')[0]
http_client = RequestsClient()
http_client.set_basic_auth(host, user, password)
client = SwaggerClient.from_url('%s/swagger.json' % nomad_url, http_client=http_client)
```
## Uploading data
Now, we can look at actually using the nomad API. The API is divided into several
modules: *uploads*, *repo*, *archive*, *raw*, etc. Each provided functionality for
a certain aspect of nomad.
The *uploads* endpoints can be used to, you guessed it, upload your data. But they
also allow to get process on the upload processing; inspect, delete, and publish uploads;
and get details about the uploaded data, which code input/output files where found, etc.
### Uploading a file
Its simple, since bravado supports uploading files:
```python
with open(upload_file, 'rb') as f:
upload = client.uploads.upload(file=f).response().result
```
If you already have you file on the nomad servers, e.g. under `/nomad/my_files/example.zip`,
you can skip the actual upload and say:
```python
upload = client.uploads.upload(local_path='/nomad/my_files/example.zip').response().result
```
### Supervising the processing
Once uploaded, nomad will extract the file, identify code data, parse and normalize the
data. We call this *processing* and *processing* consists of *tasks* (uploading, extracting, parsing).
You can consistently pull the API, to get an update on the processing and check if all
tasks have completed.
```python
while upload.tasks_running:
upload = client.uploads.get_upload(upload_id=upload.upload_id).response().result
time.sleep(5)
print('processed: %d, failures: %d' % (upload.processed_calcs, upload.failed_calcs))
```
Once there are no more tasks running, you can check if your upload was a success. If it
was not successful, you can also delete the upload again:
```python
if upload.tasks_status != 'SUCCESS':
print('something went wrong')
print('errors: %s' % str(upload.errors))
# delete the unsuccessful upload
client.uploads.delete_upload(upload_id=upload.upload_id).response().result
sys.exit(1)
```
Of course, you can also visit the nomad GUI
([http://enc-staging-nomad.esc.rzg.mpg.de/fairdi/nomad/v0.3.0/upload](http://enc-staging-nomad.esc.rzg.mpg.de/fairdi/nomad/v0.3.0/upload))
to inspect your uploads. (You might click reload, if you had the page already open.)
### Publishing your upload
The uploaded data is only visible to you. We call this *staging*. After the processing
was successful and you are satisfied with our processing, you have to publish the upload.
This also allows you to add additional meta-data to your upload (e.g. comments, references, coauthors, etc.).
Here you also determine, if you want an *embargo* on your data.
Once the data was published, you cannot delete it anymore. You can skip this step, but
the reset of the tutorial, will only work for you, because the data is only visible to you.
To initiate the publish and provide further data:
```python
client.uploads.exec_upload_operation(upload_id=upload.upload_id, payload={
'operation': 'publish',
'metadata': {
'comment': 'Data from a cool external project',
'references': ['http://external.project.eu'],
# 'coauthors': ['sheldon.cooper@ucla.edu'], this does not yet work with emails
# 'external_id': 'external_id' this does also not work, but we could implement something like this
}
})
```
Publishing, also might take a while. You can inspect this analog to the upload processing:
```python
while upload.process_running:
try:
upload = client.uploads.get_upload(upload_id=upload.upload_id).response().result
time.sleep(1)
except HTTPNotFound:
# upload gets deleted from the upload staging area once published
break
```
This time we needed some exception handling, since the upload will be removed from the
staging area, and you will get a 404 on the `uploads` endpoint.
## Searching for data
The *repo* part of the API contains a *search* endpoint that support many different
quantities to search for. These include `formula` (e.g. *AcAg*), `system` (e.g. *bulk/2D/atom*), `spacegroup`, `authors`, `code` (e.g. *VASP*), etc.
In the following example, we search for the specific path segment `AcAg`.
```python
result = client.repo.search(paths='AcAg').response().result
if result.pagination.total == 0:
print('not found')
elif result.pagination.total > 1:
print('my ids are not specific enough, bummer ... or did I uploaded stuff multiple times?')
calc = result.results[0]
print(calc)
```
The result of a search always contains the key `pagination` with pagination data (`total`, `page`, `per_page`) and `results` with an array of the search result. The search results depend on
the type of search and their is no formal swagger model for it, therefore you get plain
dictionaries.
## Downloading data
The *raw* api allows to download data. You can do that either via bravado:
```python
client.raw.get(upload_id=calc['upload_id'], path=calc['mainfile']).response()
```
In case of published data, you can also create plain URLs and use a tool like *curl*:
```python
print('%s/raw/%s/%s' % (nomad_url, calc['upload_id'], calc['mainfile']))
print('%s/raw/%s/%s/*' % (nomad_url, calc['upload_id'], os.path.dirname(calc['mainfile'])))
```
There are different options to download individual files, or zips with multiple files.
## Conclusions
This was just a small glimpse into the nomad API. You should checkout our swagger documentation
for more details on all the API endpoints and their parameters. You can explore the
API via swagger-ui and even try it in your browser. Just visit the API url.
......@@ -10,6 +10,7 @@ and infrastructure with a simplyfied architecture and consolidated code base.
introduction
setup
dev_guidelines
api_tutorial
api
ops
parser_tutorial
......
"""
This is a brief example demonstrating the public nomad@FAIRDI API for doing operations
that might be necessary to integrate external project data.
"""
from bravado.requests_client import RequestsClient
from bravado.client import SwaggerClient
from bravado.exception import HTTPNotFound
from urllib.parse import urlparse
import time
import os.path
import sys
nomad_url = 'http://enc-staging-nomad.esc.rzg.mpg.de/fairdi/nomad/v0.3.0/api'
user = 'leonard.hofstadter@nomad-fairdi.tests.de'
password = 'password'
upload_file = 'external_project_example.zip'
# create the bravado client
host = urlparse(nomad_url).netloc.split(':')[0]
http_client = RequestsClient()
http_client.set_basic_auth(host, user, password)
client = SwaggerClient.from_url('%s/swagger.json' % nomad_url, http_client=http_client)
# upload data
with open(upload_file, 'rb') as f:
upload = client.uploads.upload(file=f).response().result
while upload.tasks_running:
upload = client.uploads.get_upload(upload_id=upload.upload_id).response().result
time.sleep(5)
print('processed: %d, failures: %d' % (upload.processed_calcs, upload.failed_calcs))
# check if processing was a success
if upload.tasks_status != 'SUCCESS':
print('something went wrong')
print('errors: %s' % str(upload.errors))
# delete the unsuccessful upload
client.uploads.delete_upload(upload_id=upload.upload_id).response().result
sys.exit(1)
# publish data
client.uploads.exec_upload_operation(upload_id=upload.upload_id, payload={
'operation': 'publish',
'metadata': {
'comment': 'Data from a cool external project',
'references': ['http://external.project.eu'],
# 'coauthors': ['sheldon.cooper@ucla.edu'], this does not yet work with emails
# 'external_id': 'external_id' this does also not work, but we could implement something like this
}
}).response().result
while upload.process_running:
try:
upload = client.uploads.get_upload(upload_id=upload.upload_id).response().result
time.sleep(1)
except HTTPNotFound:
# upload gets deleted from the upload staging area once published
break
# search for data
result = client.repo.search(paths='external_id').response().result
if result.pagination.total == 0:
print('not found')
sys.exit(1)
elif result.pagination.total > 1:
print('my ids are not specific enough, bummer ... or did I uploaded stuff multiple times?')
# The results key holds an array with the current page data
calc = result.results[0]
# download data
client.raw.get(upload_id=calc['upload_id'], path=calc['mainfile']).response()
# download urls, e.g. for curl
print('%s/raw/%s/%s' % (nomad_url, calc['upload_id'], calc['mainfile']))
print('%s/raw/%s/%s/*' % (nomad_url, calc['upload_id'], os.path.dirname(calc['mainfile'])))
......@@ -7,6 +7,7 @@
"@material-ui/icons": "^3.0.2",
"@navjobs/upload": "^3.1.3",
"base-64": "^0.1.0",
"chroma-js": "^2.0.3",
"fetch": "^1.1.0",
"file-saver": "^2.0.0",
"html-to-react": "^1.3.3",
......
import React from 'react'
import PropTypes from 'prop-types'
import periodicTableData from './PeriodicTableData'
import { withStyles, Paper, Typography, Button, Tooltip } from '@material-ui/core'
import { withStyles, Typography, Button, Tooltip } from '@material-ui/core'
import chroma from 'chroma-js'
const elements = []
for (var i = 0; i < 10; i++) {
......@@ -17,7 +18,9 @@ class ElementUnstyled extends React.Component {
classes: PropTypes.object.isRequired,
element: PropTypes.object.isRequired,
onClick: PropTypes.func,
selected: PropTypes.bool
selected: PropTypes.bool,
count: PropTypes.number.isRequired,
heatmapScale: PropTypes.func.isRequired
}
static styles = theme => ({
......@@ -40,9 +43,6 @@ class ElementUnstyled extends React.Component {
minHeight: 0,
borderRadius: 0
},
contained: {
backgroundColor: 'white'
},
containedPrimary: {
backgroundColor: theme.palette.primary.main
},
......@@ -54,62 +54,51 @@ class ElementUnstyled extends React.Component {
padding: 0,
fontSize: 8
},
actinide: {
backgroundColor: '#E8EAF6'
},
alkalimetal: {
backgroundColor: '#E3F2FD'
},
alkalineearthmetal: {
backgroundColor: '#EDE7F6'
},
diatomicnonmetal: {
backgroundColor: '#F3E5F5'
},
lanthanide: {
backgroundColor: '#FCE4EC'
},
metalloid: {
backgroundColor: '#FFEBEE'
},
noblegas: {
backgroundColor: '#E0F7FA'
},
polyatomicnonmetal: {
backgroundColor: '#E0F2F1'
},
'post-transitionmetal': {
backgroundColor: '#E1F5FE'
},
transitionmetal: {
backgroundColor: '#F9FBE7'
count: {
position: 'absolute',
bottom: 2,
right: 2,
margin: 0,
padding: 0,
fontSize: 8
}
})
render() {
const {classes, element, selected} = this.props
const {classes, element, selected, count, heatmapScale} = this.props
const buttonClasses = {
root: classes.button,
contained: (!selected ? classes[element.category] : null) || classes.contained,
containedPrimary: classes.containedPrimary
}
const disabled = count <= 0
return (
<div className={classes.root}>
<Tooltip title={element.name}>
<div>
<Button
disabled={disabled}
classes={buttonClasses}
style={{backgroundColor: count > 0 && !selected ? heatmapScale(count).hex() : undefined}}
onClick={this.props.onClick} variant="contained"
color={selected ? 'primary' : 'default'}
>
{element.symbol}
</Button>
</div>
</Tooltip>
<Typography
classes={{root: classes.number}} variant="caption"
style={selected ? {color: 'white'} : {}}>
style={selected ? {color: 'white'} : disabled ? {color: '#BDBDBD'} : {}}>
{element.number}
</Typography>
{count >= 0
? <Typography
classes={{root: classes.count}} variant="caption"
style={selected ? {color: 'white'} : disabled ? {color: '#BDBDBD'} : {}}>
{count}
</Typography> : ''
}
</div>
)
}
......@@ -119,12 +108,13 @@ const Element = withStyles(ElementUnstyled.styles)(ElementUnstyled)
class PeriodicTable extends React.Component {
static propTypes = {
classes: PropTypes.object.isRequired
classes: PropTypes.object.isRequired,
aggregations: PropTypes.object,
onSelectionChanged: PropTypes.func.isRequired
}
static styles = theme => ({
root: {
padding: theme.spacing.unit * 2,
overflowX: 'scroll'
},
table: {
......@@ -144,19 +134,33 @@ class PeriodicTable extends React.Component {
onElementClicked(element) {
const index = this.state.selected.indexOf(element)
const isClicked = index >= 0
let selected
if (isClicked) {
const selected = [...this.state.selected]
selected.slice(index, 1)
selected = [...this.state.selected]
selected.splice(index, 1)
this.setState({selected: selected})
} else {
this.setState({selected: [element, ...this.state.selected]})
selected = [element, ...this.state.selected]
this.setState({selected: selected})
}
this.props.onSelectionChanged(selected.map(element => element.symbol))
}
unSelectedAggregations() {
const { aggregations } = this.props
const { selected } = this.state
return Object.keys(aggregations)
.filter(key => selected.indexOf(key) === -1)
.map(key => aggregations[key])
}
render() {
const {classes} = this.props
const {classes, aggregations} = this.props
const max = aggregations ? Math.max(...this.unSelectedAggregations()) || 1 : 1
const heatmapScale = chroma.scale(['#ffcdd2', '#d50000']).domain([1, max], 10, 'log')
return (
<Paper className={classes.root}>
<div className={classes.root}>
<table className={classes.table}>
<tbody>
{elements.map((row, i) => (
......@@ -166,6 +170,9 @@ class PeriodicTable extends React.Component {