Merge branch 'migration' into 'master'

Merge version 0.4.0

See merge request !33

.gitlab-ci.yml

@@ -72,7 +72,9 @@ linting:
     - python -m pylint --load-plugins=pylint_mongoengine nomad tests
     - python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests
   except:
-    - /^dev-.*$/
+    variables:
+      - $CI_COMMIT_REF_NAME =~ /^dev-.*$/
+      - $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
 
 tests:
   stage: test
@@ -104,7 +106,9 @@ tests:
     - cd /app
     - python -m pytest --cov=nomad -sv tests
   except:
-    - /^dev-.*$/
+    variables:
+      - $CI_COMMIT_REF_NAME =~ /^dev-.*$/
+      - $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i
 
 # does currently not work, current GitLab CI runner does not network services with each other
 # integration-tests:
@@ -207,9 +211,12 @@ deploy:
     - export FILES_PATH="/scratch/nomad-fair/fs/nomad_v${NOMAD_VERSION}"
     - if [ ${DEPLOYS}  -eq 0 ]; then
         helm install --name=${RELEASE_NAME} . --namespace=${STAGING_NAMESPACE}
+          --set api.disableReset="false"
           --set proxy.nodePort="300${NUMERIC_VERSION//./}"
           --set proxy.external.path=${EXTERNAL_PATH}
           --set dbname=${DBNAME}
+          --set worker.replicas=1
+          --set worker.memrequest=32
           --set volumes.files=${FILES_PATH};
       else
         helm upgrade ${RELEASE_NAME} . --namespace=${STAGING_NAMESPACE} --recreate-pods;

.gitmodules

@@ -51,3 +51,22 @@
 [submodule "dependencies/parsers/quantum-espresso"]
 	path = dependencies/parsers/quantum-espresso
 	url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-quantum-espresso
+[submodule "dependencies/parsers/abinit"]
+	path = dependencies/parsers/abinit
+	url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-abinit
+
+[submodule "dependencies/parsers/orca"]
+	path = dependencies/parsers/orca
+	url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-orca
+[submodule "dependencies/parsers/castep"]
+	path = dependencies/parsers/castep
+	url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-castep
+[submodule "dependencies/parsers/dl-poly"]
+	path = dependencies/parsers/dl-poly
+	url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-dl-poly
+[submodule "dependencies/parsers/lib-atoms"]
+	path = dependencies/parsers/lib-atoms
+	url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-lib-atoms
+[submodule "dependencies/parsers/octopus"]
+	path = dependencies/parsers/octopus
+	url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-octopus

.vscode/launch.json

@@ -138,16 +138,65 @@
       ]
     },
     {
-      "name": "Python: Quantum Espresso",
+      "name": "Quantum Espresso Normalizer",
       "type": "python",
       "request": "launch",
       "cwd": "${workspaceFolder}",
       "program": "${workspaceFolder}/.pyenv/bin/pytest",
       "args": [
-        "-sv", "tests/test_parsing.py::test_parser[parsers/quantumespresso-tests/data/parsers/quantum-espresso/W.out]"
+        "-sv", "tests/test_normalizing.py::test_normalizer[parsers/quantumespresso-tests/data/parsers/quantum-espresso/W.out]"
+      ]
+    },
+    {
+      "name": "Abinit Normalizer",
+      "type": "python",
+      "request": "launch",
+      "cwd": "${workspaceFolder}",
+      "program": "${workspaceFolder}/.pyenv/bin/pytest",
+      "args": [
+        "-sv", "tests/test_normalizing.py::test_normalizer[parsers/abinit-tests/data/parsers/abinit/Fe.out]"
+      ]
+    },
+    {
+      "name": "Castep Normalizer",
+      "type": "python",
+      "request": "launch",
+      "cwd": "${workspaceFolder}",
+      "program": "${workspaceFolder}/.pyenv/bin/pytest",
+      "args": [
+        "-sv", "tests/test_normalizing.py::test_normalizer[parsers/castep-tests/data/parsers/castep/BC2N-Pmm2-Raman.castep]"
+      ]
+    },
+    {
+      "name": "DL-Poly Normalizer",
+      "type": "python",
+      "request": "launch",
+      "cwd": "${workspaceFolder}",
+      "program": "${workspaceFolder}/.pyenv/bin/pytest",
+      "args": [
+        "-sv", "tests/test_normalizing.py::test_normalizer[parsers/dl-poly-tests/data/parsers/dl-poly/OUTPUT]"
+      ]
+    },
+    {
+      "name": "Lib Atoms Normalizer",
+      "type": "python",
+      "request": "launch",
+      "cwd": "${workspaceFolder}",
+      "program": "${workspaceFolder}/.pyenv/bin/pytest",
+      "args": [
+        "-sv", "tests/test_normalizing.py::test_normalizer[parsers/lib-atoms-tests/data/parsers/lib-atoms/gp.xml]"
+      ]
+    },
+    {
+      "name": "Octopus Normalizer",
+      "type": "python",
+      "request": "launch",
+      "cwd": "${workspaceFolder}",
+      "program": "${workspaceFolder}/.pyenv/bin/pytest",
+      "args": [
+        "-sv", "tests/test_normalizing.py::test_normalizer[parsers/octopus-tests/data/parsers/octopus/stdout.txt]"
       ]
     },
-
     {
       "name": "Python: Current File",
       "type": "python",

nomad-meta-info @ b232a5fd

-Subproject commit 8957235e5ca6bbb3278790e2587a30323169610f
+Subproject commit b232a5fdd11c344656ff1e6ec7a2ed6f787cf49f

abinit @ 18f8ccde

+Subproject commit 18f8ccde48f39a9237882064e20e5a1e7baff5c0

band @ 71f7a2ad

-Subproject commit b68bce7db5c02ac56b1cf892ae20458fffb23b65
+Subproject commit 71f7a2ad0d77d376e30d3c1a8ae55920fc7d7e5c

castep @ 5c7cf8ad

+Subproject commit 5c7cf8add659c51d940960daa430927923720f7e

dl-poly @ de6d49f8

+Subproject commit de6d49f89748010236baa9b683464b70bc8507a6

lib-atoms @ d7bfc62d

+Subproject commit d7bfc62deebfd40d3033817c529d1f6f89c901e5

octopus @ 7de9f912

+Subproject commit 7de9f91273f4c5becaaad7fef8710e68b3cd63de

orca @ 83a4de4e

+Subproject commit 83a4de4efa40731f7e192c795fc1e82f00f27fc8

python_common @ fcda3a9a

-Subproject commit 2a59119a6635454eba33c268d8cf8caa6eecd204
+Subproject commit fcda3a9ab4a23ea0c7363955999943d9be6fa0fb

docs/api_tutorial.md

+# API Tutorial
+
+This tutorial assumes that you want to
+
+- upload some data
+- publish the data
+- find it
+- download it again
+
+## Prequisites
+
+### Python
+The tutorial was tested with Python 3, but it might as well work with Python 2.
+
+### Python packages
+We do not assume many specific python packages. Only the *bravado* package (available
+via pipy) is required. It allows us to use the nomad ReST API in a more friendly and
+pythonic way. You can simply install it the usual way
+
+```
+pip install bravado
+```
+
+For the following code snippets, we need the following imports:
+
+```python
+from bravado.requests_client import RequestsClient
+from bravado.client import SwaggerClient
+from bravado.exception import HTTPNotFound
+from urllib.parse import urlparse
+import time
+import os.path
+import sys
+```
+
+### An example file
+Lets assume you have an example upload file ready. Its a `.zip` (`.tgz` would also work)
+with some *VASP* data from a single run at `/example/AcAg/vasprun.xml`, `/example/AcAg/OUTCAR`, ...
+Lets keep the filename in a variable:
+
+```python
+upload_file = 'example.zip'
+```
+
+### Nomad
+We need to know the nomad installation to use and its respective API URL. To upload
+data you also need an account (email, password):
+
+```python
+nomad_url = 'http://enc-staging-nomad.esc.rzg.mpg.de/fairdi/nomad/v0.3.0/api'
+user = 'leonard.hofstadter@nomad-fairdi.tests.de'
+password = 'password'
+```
+
+### Using bravado
+Bravado reads a ReST API's definition from a `swagger.json` as it is provided by
+many APIs, including nomad's of course. Bravado also allows to use authentication,
+which makes it even easier. The following would be a typical setup:
+
+```python
+host = urlparse(nomad_url).netloc.split(':')[0]
+http_client = RequestsClient()
+http_client.set_basic_auth(host, user, password)
+client = SwaggerClient.from_url('%s/swagger.json' % nomad_url, http_client=http_client)
+```
+
+## Uploading data
+Now, we can look at actually using the nomad API. The API is divided into several
+modules: *uploads*, *repo*, *archive*, *raw*, etc. Each provided functionality for
+a certain aspect of nomad.
+
+The *uploads* endpoints can be used to, you guessed it, upload your data. But they
+also allow to get process on the upload processing; inspect, delete, and publish uploads;
+and get details about the uploaded data, which code input/output files where found, etc.
+
+### Uploading a file
+
+Its simple, since bravado supports uploading files:
+
+```python
+with open(upload_file, 'rb') as f:
+    upload = client.uploads.upload(file=f).response().result
+```
+
+If you already have you file on the nomad servers, e.g. under `/nomad/my_files/example.zip`,
+you can skip the actual upload and say:
+```python
+upload = client.uploads.upload(local_path='/nomad/my_files/example.zip').response().result
+```
+
+### Supervising the processing
+
+Once uploaded, nomad will extract the file, identify code data, parse and normalize the
+data. We call this *processing* and *processing* consists of *tasks* (uploading, extracting, parsing).
+You can consistently pull the API, to get an update on the processing and check if all
+tasks have completed.
+
+```python
+while upload.tasks_running:
+    upload = client.uploads.get_upload(upload_id=upload.upload_id).response().result
+    time.sleep(5)
+    print('processed: %d, failures: %d' % (upload.processed_calcs, upload.failed_calcs))
+```
+
+Once there are no more tasks running, you can check if your upload was a success. If it
+was not successful, you can also delete the upload again:
+```python
+if upload.tasks_status != 'SUCCESS':
+    print('something went wrong')
+    print('errors: %s' % str(upload.errors))
+
+    # delete the unsuccessful upload
+    client.uploads.delete_upload(upload_id=upload.upload_id).response().result
+    sys.exit(1)
+```
+
+Of course, you can also visit the nomad GUI
+([http://enc-staging-nomad.esc.rzg.mpg.de/fairdi/nomad/v0.3.0/upload](http://enc-staging-nomad.esc.rzg.mpg.de/fairdi/nomad/v0.3.0/upload))
+to inspect your uploads. (You might click reload, if you had the page already open.)
+
+
+### Publishing your upload
+The uploaded data is only visible to you. We call this *staging*. After the processing
+was successful and you are satisfied with our processing, you have to publish the upload.
+This also allows you to add additional meta-data to your upload (e.g. comments, references, coauthors, etc.).
+Here you also determine, if you want an *embargo* on your data.
+
+Once the data was published, you cannot delete it anymore. You can skip this step, but
+the reset of the tutorial, will only work for you, because the data is only visible to you.
+
+To initiate the publish and provide further data:
+```python
+client.uploads.exec_upload_operation(upload_id=upload.upload_id, payload={
+    'operation': 'publish',
+    'metadata': {
+        'comment': 'Data from a cool external project',
+        'references': ['http://external.project.eu'],
+        # 'coauthors': ['sheldon.cooper@ucla.edu'],  this does not yet work with emails
+        # 'external_id': 'external_id'  this does also not work, but we could implement something like this
+    }
+})
+```
+
+Publishing, also might take a while. You can inspect this analog to the upload processing:
+
+```python
+while upload.process_running:
+    try:
+        upload = client.uploads.get_upload(upload_id=upload.upload_id).response().result
+        time.sleep(1)
+    except HTTPNotFound:
+        # upload gets deleted from the upload staging area once published
+        break
+```
+
+This time we needed some exception handling, since the upload will be removed from the
+staging area, and you will get a 404 on the `uploads` endpoint.
+
+
+## Searching for data
+The *repo* part of the API contains a *search* endpoint that support many different
+quantities to search for. These include `formula` (e.g. *AcAg*), `system` (e.g. *bulk/2D/atom*), `spacegroup`, `authors`, `code` (e.g. *VASP*), etc.
+In the following example, we search for the specific path segment `AcAg`.
+
+```python
+result = client.repo.search(paths='AcAg').response().result
+if result.pagination.total == 0:
+    print('not found')
+elif result.pagination.total > 1:
+    print('my ids are not specific enough, bummer ... or did I uploaded stuff multiple times?')
+calc = result.results[0]
+print(calc)
+```
+
+The result of a search always contains the key `pagination` with pagination data (`total`, `page`, `per_page`) and `results` with an array of the search result. The search results depend on
+the type of search and their is no formal swagger model for it, therefore you get plain
+dictionaries.
+
+
+## Downloading data
+The *raw* api allows to download data. You can do that either via bravado:
+```python
+client.raw.get(upload_id=calc['upload_id'], path=calc['mainfile']).response()
+```
+
+In case of published data, you can also create plain URLs and use a tool like *curl*:
+```python
+print('%s/raw/%s/%s' % (nomad_url, calc['upload_id'], calc['mainfile']))
+print('%s/raw/%s/%s/*' % (nomad_url, calc['upload_id'], os.path.dirname(calc['mainfile'])))
+```
+
+There are different options to download individual files, or zips with multiple files.
+
+## Conclusions
+This was just a small glimpse into the nomad API. You should checkout our swagger documentation
+for more details on all the API endpoints and their parameters. You can explore the
+API via swagger-ui and even try it in your browser. Just visit the API url.

docs/index.rst

@@ -10,6 +10,7 @@ and infrastructure with a simplyfied architecture and consolidated code base.
    introduction
    setup
    dev_guidelines
+   api_tutorial
    api
    ops
    parser_tutorial

examples/externalproject.py

+"""
+This is a brief example demonstrating the public nomad@FAIRDI API for doing operations
+that might be necessary to integrate external project data.
+"""
+
+from bravado.requests_client import RequestsClient
+from bravado.client import SwaggerClient
+from bravado.exception import HTTPNotFound
+from urllib.parse import urlparse
+import time
+import os.path
+import sys
+
+nomad_url = 'http://enc-staging-nomad.esc.rzg.mpg.de/fairdi/nomad/v0.3.0/api'
+user = 'leonard.hofstadter@nomad-fairdi.tests.de'
+password = 'password'
+
+upload_file = 'external_project_example.zip'
+
+# create the bravado client
+host = urlparse(nomad_url).netloc.split(':')[0]
+http_client = RequestsClient()
+http_client.set_basic_auth(host, user, password)
+client = SwaggerClient.from_url('%s/swagger.json' % nomad_url, http_client=http_client)
+
+# upload data
+with open(upload_file, 'rb') as f:
+    upload = client.uploads.upload(file=f).response().result
+while upload.tasks_running:
+    upload = client.uploads.get_upload(upload_id=upload.upload_id).response().result
+    time.sleep(5)
+    print('processed: %d, failures: %d' % (upload.processed_calcs, upload.failed_calcs))
+
+#   check if processing was a success
+if upload.tasks_status != 'SUCCESS':
+    print('something went wrong')
+    print('errors: %s' % str(upload.errors))
+    # delete the unsuccessful upload
+    client.uploads.delete_upload(upload_id=upload.upload_id).response().result
+    sys.exit(1)
+
+# publish data
+client.uploads.exec_upload_operation(upload_id=upload.upload_id, payload={
+    'operation': 'publish',
+    'metadata': {
+        'comment': 'Data from a cool external project',
+        'references': ['http://external.project.eu'],
+        # 'coauthors': ['sheldon.cooper@ucla.edu'],  this does not yet work with emails
+        # 'external_id': 'external_id'  this does also not work, but we could implement something like this
+    }
+}).response().result
+while upload.process_running:
+    try:
+        upload = client.uploads.get_upload(upload_id=upload.upload_id).response().result
+        time.sleep(1)
+    except HTTPNotFound:
+        # upload gets deleted from the upload staging area once published
+        break
+
+# search for data
+result = client.repo.search(paths='external_id').response().result
+if result.pagination.total == 0:
+    print('not found')
+    sys.exit(1)
+elif result.pagination.total > 1:
+    print('my ids are not specific enough, bummer ... or did I uploaded stuff multiple times?')
+# The results key holds an array with the current page data
+calc = result.results[0]
+
+# download data
+client.raw.get(upload_id=calc['upload_id'], path=calc['mainfile']).response()
+
+# download urls, e.g. for curl
+print('%s/raw/%s/%s' % (nomad_url, calc['upload_id'], calc['mainfile']))
+print('%s/raw/%s/%s/*' % (nomad_url, calc['upload_id'], os.path.dirname(calc['mainfile'])))

gui/package.json

@@ -7,6 +7,7 @@
     "@material-ui/icons": "^3.0.2",
     "@navjobs/upload": "^3.1.3",
     "base-64": "^0.1.0",
+    "chroma-js": "^2.0.3",
     "fetch": "^1.1.0",
     "file-saver": "^2.0.0",
     "html-to-react": "^1.3.3",

gui/src/components/PeriodicTable.js

 import React from 'react'
 import PropTypes from 'prop-types'
 import periodicTableData from './PeriodicTableData'
-import { withStyles, Paper, Typography, Button, Tooltip } from '@material-ui/core'
+import { withStyles, Typography, Button, Tooltip } from '@material-ui/core'
+import chroma from 'chroma-js'
 
 const elements = []
 for (var i = 0; i < 10; i++) {
@@ -17,7 +18,9 @@ class ElementUnstyled extends React.Component {
     classes: PropTypes.object.isRequired,
     element: PropTypes.object.isRequired,
     onClick: PropTypes.func,
-    selected: PropTypes.bool
+    selected: PropTypes.bool,
+    count: PropTypes.number.isRequired,
+    heatmapScale: PropTypes.func.isRequired
   }
 
   static styles = theme => ({
@@ -40,9 +43,6 @@ class ElementUnstyled extends React.Component {
       minHeight: 0,
       borderRadius: 0
     },
-    contained: {
-      backgroundColor: 'white'
-    },
     containedPrimary: {
       backgroundColor: theme.palette.primary.main
     },
@@ -54,62 +54,51 @@ class ElementUnstyled extends React.Component {
       padding: 0,
       fontSize: 8
     },
-    actinide: {
-      backgroundColor: '#E8EAF6'
-    },
-    alkalimetal: {
-      backgroundColor: '#E3F2FD'
-    },
-    alkalineearthmetal: {
-      backgroundColor: '#EDE7F6'
-    },
-    diatomicnonmetal: {
-      backgroundColor: '#F3E5F5'
-    },
-    lanthanide: {
-      backgroundColor: '#FCE4EC'
-    },
-    metalloid: {
-      backgroundColor: '#FFEBEE'
-    },
-    noblegas: {
-      backgroundColor: '#E0F7FA'
-    },
-    polyatomicnonmetal: {
-      backgroundColor: '#E0F2F1'
-    },
-    'post-transitionmetal': {
-      backgroundColor: '#E1F5FE'
-    },
-    transitionmetal: {
-      backgroundColor: '#F9FBE7'
+    count: {
+      position: 'absolute',
+      bottom: 2,
+      right: 2,
+      margin: 0,
+      padding: 0,
+      fontSize: 8
     }
   })
 
   render() {
-    const {classes, element, selected} = this.props
+    const {classes, element, selected, count, heatmapScale} = this.props
     const buttonClasses = {
       root: classes.button,
-      contained: (!selected ? classes[element.category] : null) || classes.contained,
       containedPrimary: classes.containedPrimary
     }
+    const disabled = count <= 0
 
     return (
       <div className={classes.root}>
         <Tooltip title={element.name}>
-          <Button
-            classes={buttonClasses}
-            onClick={this.props.onClick} variant="contained"
-            color={selected ? 'primary' : 'default'}
-          >
-            {element.symbol}
-          </Button>
+          <div>
+            <Button
+              disabled={disabled}
+              classes={buttonClasses}
+              style={{backgroundColor: count > 0 && !selected ? heatmapScale(count).hex() : undefined}}
+              onClick={this.props.onClick} variant="contained"
+              color={selected ? 'primary' : 'default'}
+            >
+              {element.symbol}
+            </Button>
+          </div>
         </Tooltip>
         <Typography
           classes={{root: classes.number}} variant="caption"
-          style={selected ? {color: 'white'} : {}}>
+          style={selected ? {color: 'white'} : disabled ? {color: '#BDBDBD'} : {}}>
           {element.number}
         </Typography>
+        {count >= 0
+          ? <Typography
+            classes={{root: classes.count}} variant="caption"
+            style={selected ? {color: 'white'} : disabled ? {color: '#BDBDBD'} : {}}>
+            {count}
+          </Typography> : ''
+        }
       </div>
     )
   }
@@ -119,12 +108,13 @@ const Element = withStyles(ElementUnstyled.styles)(ElementUnstyled)
 
 class PeriodicTable extends React.Component {
   static propTypes = {
-    classes: PropTypes.object.isRequired
+    classes: PropTypes.object.isRequired,
+    aggregations: PropTypes.object,
+    onSelectionChanged: PropTypes.func.isRequired
   }
 
   static styles = theme => ({
     root: {
-      padding: theme.spacing.unit * 2,
       overflowX: 'scroll'
     },
     table: {
@@ -144,19 +134,33 @@ class PeriodicTable extends React.Component {
   onElementClicked(element) {
     const index = this.state.selected.indexOf(element)
     const isClicked = index >= 0
+    let selected
     if (isClicked) {
-      const selected = [...this.state.selected]
-      selected.slice(index, 1)
+      selected = [...this.state.selected]
+      selected.splice(index, 1)
       this.setState({selected: selected})
     } else {
-      this.setState({selected: [element, ...this.state.selected]})
+      selected = [element, ...this.state.selected]
+      this.setState({selected: selected})
     }
+
+    this.props.onSelectionChanged(selected.map(element => element.symbol))
+  }
+
+  unSelectedAggregations() {
+    const { aggregations } = this.props
+    const { selected } = this.state
+    return Object.keys(aggregations)
+      .filter(key => selected.indexOf(key) === -1)
+      .map(key => aggregations[key])
   }
 
   render() {
-    const {classes} = this.props
+    const {classes, aggregations} = this.props
+    const max = aggregations ? Math.max(...this.unSelectedAggregations()) || 1 : 1
+    const heatmapScale = chroma.scale(['#ffcdd2', '#d50000']).domain([1, max], 10, 'log')