Merge branch 'v0.9.0' into 'master'

V0.9.0

See merge request !188

docs/dev/setup.md

@@ -121,9 +121,13 @@ The NOMAD GUI requires static artifacts that are generated from the NOMAD Python
 ```
 nomad dev metainfo > gui/src/metainfo.json
 nomad dev searchQuantities > gui/src/searchQuantities.json
+nomad dev units > gui/src/units.js
 ./gitinfo.sh
 ```
 
+In additional, you have to do some more steps to prepare your working copy to run all
+the tests. See below.
+
 ## Build and run the infrastructure with docker
 
 ### Docker and nomad
@@ -218,6 +222,33 @@ yarn start
 ```
 
 ## Run the tests
+
+### additional settings and artifacts
+To run the tests some additional settings and files are necessary that are not part
+of the code base.
+
+First you need to create a `nomad.yaml` with the admin password for the user management
+system:
+```
+keycloak:
+  password: <the-password>
+```
+
+Secondly, you need to provide the `springer.msg` Springer materials database. It can
+be copied from `/nomad/fairdi/db/data/springer.msg` on our servers and should
+be placed at `nomad/normalizing/data/springer.msg`.
+
+Thirdly, you have to provide static files to serve the docs and NOMAD distribution:
+```
+cd docs
+make html
+cd ..
+python setup.py compile
+python setup.py sdist
+cp dist/nomad-lab-*.tar.gz dist/nomad-lab.tar.gz
+```
+
+### run the necessary infrastructure
 You need to have the infrastructure partially running: elastic, rabbitmq.
 The rest should be mocked or provided by the tests. Make sure that you do no run any
 worker, as they will fight for tasks in the queue.
@@ -266,7 +297,7 @@ line size ruler, etc.
     "git.enableSmartCommit": true,
     "eslint.autoFixOnSave": true,
     "python.linting.pylintArgs": [
-        "--load-plugins=pylint_mongoengine",
+        "--load-plugins=pylint_mongoengine,nomad/metainfo/pylint_plugin",
     ],
     "python.linting.pep8Path": "pycodestyle",
     "python.linting.pep8Enabled": true,

examples/archive/client.py

@@ -2,33 +2,40 @@
 A simple example that uses the NOMAD client library to access the archive.
 '''
 
-from nomad import config
 from nomad.client import ArchiveQuery
 from nomad.metainfo import units
 
 
 query = ArchiveQuery(
+    # url='http://nomad-lab.eu/prod/rae/beta/api',
     query={
-        'dft.compound_type': 'binary',
-        'dft.crystal_system': 'cubic',
-        'dft.code_name': 'FHI-aims',
-        'atoms': ['O']
+        '$and': [
+            {'dft.code_name': 'VASP'},
+            {'$gte': {'n_atoms': 3}},
+            {'$lte': {'dft.workflow.section_relaxation.final_energy_difference': 1e-24}}
+        ]
     },
     required={
-        'section_run[0]': {
-            'section_single_configuration_calculation[-2]': {
-                'energy_total': '*'
-            },
-            'section_system[-2]': '*'
+        'section_workflow': {
+            'section_relaxation': {
+                'final_calculation_ref': {
+                    'energy_total': '*',
+                    'single_configuration_calculation_to_system_ref': {
+                        'chemical_composition_reduced': '*'
+                    }
+                }
+            }
         }
     },
-    per_page=10,
-    max=1000)
+    parallel=10,
+    per_page=1000,
+    max=10000)
 
-print(query)
+for i, result in enumerate(query):
+    if i < 10:
+        calc = result.section_workflow.section_relaxation.final_calculation_ref
+        energy = calc.energy_total
+        formula = calc.single_configuration_calculation_to_system_ref.chemical_composition_reduced
+        print('%s: energy %s' % (formula, energy.to(units.hartree)))
 
-for result in query[0:10]:
-    run = result.section_run[0]
-    energy = run.section_single_configuration_calculation[0].energy_total
-    formula = run.section_system[0].chemical_composition_reduced
-    print('%s: energy %s' % (formula, energy.to(units.hartree)))
+print(query)

examples/parse.py

@@ -2,12 +2,12 @@ import sys
 from nomad.cli.parse import parse, normalize_all
 
 # match and run the parser
-backend = parse(sys.argv[1])
+archive = parse(sys.argv[1])
 # run all normalizers
-normalize_all(backend)
+normalize_all(archive)
 
 # get the 'main section' section_run as a metainfo object
-section_run = backend.resource.contents[0].section_run[0]
+section_run = archive.section_run[0]
 
 # get the same data as JSON serializable Python dict
 python_dict = section_run.m_to_dict()

examples/tutorials/api_with_archive_query.py

+'''
+A simple example used in the NOMAD webinar API tutorial
+'''
+
+from nomad.client import ArchiveQuery
+
+query = ArchiveQuery(
+    url='http://labdev-nomad.esc.rzg.mpg.de/fairdi/nomad/reprocess/api',
+    query={
+        'dft.code_name': 'VASP',
+        'atoms': ['Ti', 'O']
+    },
+    required={
+        'section_run': {
+            'section_single_configuration_calculation[-1]': {
+                'energy_total': '*',
+                'section_dos': '*'
+            }
+        }
+    },
+    parallel=1,
+    max=10)
+
+print(query)
+result = query[0]
+print(result.section_run[0].section_single_configuration_calculation[-1].section_dos[0].dos_energies)

examples/tutorials/api_with_requests.py

+# type: ignore
+
+'''
+A simple example used in the NOMAD webinar API tutorial
+'''
+
+import requests
+import json
+
+base_url = 'http://nomad-lab.eu/prod/rae/api'
+
+# response = requests.get(base_url + '/repo?datasets.name=NOMAD%20webinar')
+response = requests.get(
+    base_url + '/repo',
+    params={'datasets.name': 'NOMAD webinar', 'per_page': 1})
+
+data = response.json()
+upload_id = data['results'][0]['upload_id']
+calc_id = data['results'][0]['calc_id']
+
+response = requests.get(
+    base_url + '/archive/%s/%s' % (upload_id, calc_id))
+
+print(json.dumps(response.json(), indent=2))
+print(
+    response.json()['section_run'][0]['section_single_configuration_calculation'][-1]['section_dos'][0]['dos_energies'])

gui/package.json

 {
   "name": "nomad-fair-gui",
-  "version": "0.8.9",
+  "version": "0.9.0",
   "commit": "e98694e",
   "private": true,
   "dependencies": {
+    "@lauri-codes/materia": "0.0.6",
     "@material-ui/core": "^4.0.0",
     "@material-ui/icons": "^4.0.0",
     "@material-ui/lab": "^4.0.0-alpha.49",
@@ -23,10 +24,12 @@
     "lodash": "^4.17.15",
     "material-ui-chip-input": "^1.0.0-beta.14",
     "material-ui-flat-pagination": "^4.0.0",
+    "mathjs": "^7.1.0",
     "object-hash": "^2.0.3",
     "pace": "^0.0.4",
     "pace-js": "^1.0.2",
     "piwik-react-router": "^0.12.1",
+    "plotly.js-cartesian-dist-min": "^1.54.7",
     "qs": "^6.8.0",
     "react": "^16.13.1",
     "react-app-polyfill": "^1.0.1",
@@ -51,7 +54,6 @@
     "remark": "^12.0.1",
     "remark-math": "^2.0.1",
     "swagger-client": "^3.8.22",
-    "three.js": "^0.77.1",
     "url-parse": "^1.4.3",
     "use-query-params": "^0.6.0"
   },

gui/public/env.js

@@ -2,15 +2,17 @@ window.nomadEnv = {
   'keycloakBase': 'https://nomad-lab.eu/fairdi/keycloak/auth/',
   'keycloakRealm': 'fairdi_nomad_test',
   'keycloakClientId': 'nomad_gui_dev',
+  'appBase': 'http://nomad-lab.eu/prod/rae/beta',
   'appBase': 'http://localhost:8000/fairdi/nomad/latest',
   'debug': false,
   'matomoEnabled': false,
   'matomoUrl': 'https://nomad-lab.eu/fairdi/stat',
   'matomoSiteId': '2',
   'version': {
-    'label': '0.8.8',
+    'label': '0.9.0',
     'isBeta': false,
-    'usesBetaData': false,
+    'isTest': true,
+    'usesBetaData': true,
     'officialUrl': 'https://nomad-lab.eu/prod/rae/gui'
   },
   'encyclopediaEnabled': true,

gui/public/index.html

@@ -5,6 +5,9 @@
       window.paceOptions = {
         restartOnPushState: true
       }
+      // This needs to be defined for Plotly.js graphs, see
+      // https://github.com/plotly/plotly.js/blob/master/dist/README.md#partial-bundles
+      window.PlotlyConfig = {MathJaxConfig: 'local'}
     </script>
     <script src="https://unpkg.com/pace-js@1.0.2/pace.min.js"></script>
     <link href="%PUBLIC_URL%/pace.css" rel="stylesheet" />
@@ -38,7 +41,6 @@
 
     <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
     <script id="MathJax-script" async src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
-
     <title>NOMAD</title>
   </head>
   <body>

gui/src/components/About.js

-import React, { useContext, useLayoutEffect, useRef, useCallback, useEffect } from 'react'
+import React, { useContext, useLayoutEffect, useRef, useCallback, useEffect, useState } from 'react'
 import {ReactComponent as AboutSvg} from './about.svg'
 import PropTypes from 'prop-types'
 import Markdown from './Markdown'
-import { appBase, optimadeBase, apiBase, debug, consent } from '../config'
+import { appBase, optimadeBase, apiBase, debug, consent, aitoolkitEnabled, encyclopediaEnabled } from '../config'
 import { apiContext } from './api'
 import packageJson from '../../package.json'
 import { domains } from './domains'
-import { Grid, Card, CardContent, Typography, makeStyles, Link } from '@material-ui/core'
+import { Grid, Card, CardContent, Typography, makeStyles, Link, Dialog, DialogTitle, DialogContent, DialogActions, Button } from '@material-ui/core'
 import { Link as RouterLink, useHistory } from 'react-router-dom'
+import tutorials from '../toolkitMetadata'
+import parserMetadata from '../parserMetadata'
 
-export const CodeList = () => {
-  const {info} = useContext(apiContext)
+function CodeInfo({code, ...props}) {
+  if (!code) {
+    return ''
+  }
 
-  if (!info) {
-    return '...'
+  const metadata = parserMetadata[code]
+
+  let introduction = `
+    For [${metadata.codeLabel || code}](${metadata.codeUrl}) please provide
+    all files that were used as input, were output by the code, or were produced you.
+  `
+  if (metadata.tableOfFiles && metadata.tableOfFiles !== '') {
+    introduction = `
+      For [${metadata.codeLabel || code}](${metadata.codeUrl}) please provide at least
+      the files from the following table (if applicable). Ideally, you upload
+      all files that were used as input, were output by the code, or were produced you.
+    `
   }
 
-  return info.codes.reduce((result, code, index) => {
-    if (index !== 0) {
-      result.push(', ')
+  return <Dialog open={true} {...props}>
+    <DialogTitle>{metadata.codeLabel || code}</DialogTitle>
+    <DialogContent>
+      <Markdown>{`
+        ${introduction} NOMAD will present all files in the same directory for each
+        recognized calculation. This works best, if you put all files that belong to
+        individual code runs into individual directories or only combine files from a few
+        runs in the same directory.
+
+        ${metadata.tableOfFiles}
+
+        ${(metadata.parserSpecific && metadata.parserSpecific !== '' &&
+        `Please note specifically for ${metadata.codeLabel || code}: ${metadata.parserSpecific}`) || ''}
+
+        To create an upload with all calculations in a directory structure:
+
+        \`\`\`
+        zip -r <upload-file>.zip <directory>/*
+        \`\`\`
+
+        You can find further information on [the project page for NOMAD's ${metadata.codeLabel || code} parser](${metadata.parserGitUrl}).
+      `}</Markdown>
+    </DialogContent>
+    <DialogActions>
+      <Button onClick={() => props.onClose && props.onClose()} color="primary">
+        close
+      </Button>
+    </DialogActions>
+  </Dialog>
+}
+CodeInfo.propTypes = {
+  code: PropTypes.string,
+  onClose: PropTypes.func
+}
+
+export const CodeList = ({withUploadInstructions}) => {
+  const [selected, setSelected] = useState(null)
+
+  const codes = Object.keys(parserMetadata).map(code => {
+    const metadata = parserMetadata[code]
+    if (!metadata) {
+      return code
     }
-    if (code.code_homepage) {
-      result.push(<Link target="external" key={code.code_name} href={code.code_homepage}>{code.code_name}</Link>)
-    } else {
-      result.push(code.code_name)
+
+    if (withUploadInstructions) {
+      return <Link
+        href="#" key={code} onClick={() => setSelected(code)}
+      >{code.codeLabel || code}</Link>
     }
-    return result
+
+    if (metadata.codeUrl) {
+      return <Link href={metadata.codeUrl} key={code} target="code">{code.codeLabel || code}</Link>
+    }
+
+    return code
+  })
+
+  const toRender = codes.reduce((list, value, index) => {
+    if (index !== 0) {
+      list.push(', ')
+    }
+    list.push(value)
+    return list
   }, [])
+
+  return <React.Fragment>
+    {toRender}
+    <CodeInfo code={selected} onClose={() => setSelected(null)} />
+  </React.Fragment>
+}
+CodeList.propTypes = {
+  withUploadInstructions: PropTypes.bool
 }
 
 const useCardStyles = makeStyles(theme => ({
@@ -97,10 +172,18 @@ export default function About() {
       history.push('/upload')
     })
     makeClickable('encyclopedia', () => {
-      window.location.href = 'https://encyclopedia.nomad-coe.eu/gui/#/search'
+      if (encyclopediaEnabled) {
+        window.location.href = `${appBase}/encyclopedia`
+      } else {
+        window.location.href = 'https://encyclopedia.nomad-coe.eu/gui/#/search'
+      }
     })
-    makeClickable('analytics', () => {
-      window.location.href = 'https://nomad-lab.eu/AItutorials'
+    makeClickable('toolkit', () => {
+      if (aitoolkitEnabled) {
+        history.push('/aitoolkit')
+      } else {
+        window.location.href = 'https://nomad-lab.eu/tools/AItutorials'
+      }
     })
     makeClickable('search', () => {
       history.push('/search')
@@ -109,16 +192,19 @@ export default function About() {
 
   useEffect(() => {
     const statistics = (info && info.statistics) || {}
+    statistics.n_tutorials = tutorials.tutorials.length
     const value = (key, unit) => {
       const nominal = statistics[key]
       let stringValue = null
       if (nominal) {
         if (nominal >= 1.0e+9) {
-          stringValue = Math.floor(nominal / 1.0e+9) + ' bln.'
+          stringValue = (nominal / 1.0e+9).toFixed(1) + ' billion'
         } else if (nominal >= 1.0e+6) {
-          stringValue = Math.floor(nominal / 1.0e+6) + ' mln.'
+          stringValue = (nominal / 1.0e+6).toFixed(1) + ' million'
+        } else if (nominal >= 1.0e+3) {
+          stringValue = (nominal / 1.0e+3).toFixed(1) + ' thousand'
         } else {
-          stringValue = Math.floor(nominal / 1.0e+3) + ' tsd.'
+          stringValue = nominal.toString()
         }
         return `${stringValue || '...'} ${unit}`
       } else {
@@ -133,6 +219,12 @@ export default function About() {
       value('n_calculations', 'results'),
       value('n_quantities', 'quantities')
     ])
+    setText('encStats', [
+      value('n_materials', 'materials')
+    ])
+    setText('toolkitStats', [
+      value('n_tutorials', 'notebooks')
+    ])
   }, [svg, info, setText])
 
   return <div className={classes.root}>

gui/src/components/ApiDialogButton.js

@@ -58,6 +58,8 @@ export function ApiDialog({title, data, onClose, ...dialogProps}) {
       <DialogTitle>{title || 'API Code'}</DialogTitle>
 
       <DialogContent classes={{root: classes.content}}>
+        { data.code && data.code.repo_url &&
+          renderCode(<span>URL to this query on the repository API:</span>, data.code.repo_url)}
         { data.code && data.code.curl &&
           renderCode(<span>Access the archive as JSON via <i>curl</i>:</span>, data.code.curl)}
         { data.code && data.code.python &&

gui/src/components/App.js

@@ -5,13 +5,14 @@ import classNames from 'classnames'
 import { MuiThemeProvider, withStyles, makeStyles } from '@material-ui/core/styles'
 import { LinearProgress, MenuList, Typography,
   AppBar, Toolbar, Button, DialogContent, DialogTitle, DialogActions, Dialog, Tooltip,
-  Snackbar, SnackbarContent, FormGroup, FormControlLabel, Switch, IconButton } from '@material-ui/core'
+  Snackbar, SnackbarContent, FormGroup, FormControlLabel, Switch, IconButton, Link as MuiLink } from '@material-ui/core'
 import { Route, Link, withRouter, useLocation } from 'react-router-dom'
 import BackupIcon from '@material-ui/icons/Backup'
 import SearchIcon from '@material-ui/icons/Search'
 import UserDataIcon from '@material-ui/icons/AccountCircle'
 import AboutIcon from '@material-ui/icons/Home'
 import FAQIcon from '@material-ui/icons/QuestionAnswer'
+import EncyclopediaIcon from '@material-ui/icons/Language'
 import MetainfoIcon from '@material-ui/icons/Info'
 import DocIcon from '@material-ui/icons/Help'
 import CodeIcon from '@material-ui/icons/Code'
@@ -25,7 +26,7 @@ import { ErrorSnacks, withErrors } from './errors'
 import { help as entryHelp, default as EntryPage } from './entry/EntryPage'
 import About from './About'
 import LoginLogout from './LoginLogout'
-import { guiBase, consent, nomadTheme, appBase, version, oasis, aitoolkitEnabled } from '../config'
+import { guiBase, consent, nomadTheme, appBase, version, oasis, aitoolkitEnabled, encyclopediaEnabled } from '../config'
 import packageJson from '../../package.json'
 import {help as uploadHelp, default as UploadPage} from './uploads/UploadPage'
 import ResolvePID from './entry/ResolvePID'
@@ -39,7 +40,7 @@ import {matomo} from '../index'
 import { useCookies } from 'react-cookie'
 import Markdown from './Markdown'
 import { help as metainfoHelp, MetainfoPage } from './archive/MetainfoBrowser'
-import AnalyticsPage from './analytics/AnalyticsPage'
+import AIToolkitPage from './aitoolkit/AIToolkitPage'
 
 export const ScrollContext = React.createContext({scrollParentRef: null})
 
@@ -126,7 +127,7 @@ function BetaSnack() {
     return ''
   }
 
-  if (!version.isBeta) {
+  if (!version.isBeta && !version.isTest) {
     return ''
   }
 
@@ -140,9 +141,9 @@ function BetaSnack() {
     <SnackbarContent
       className={classes.snack}
       message={<span style={{color: 'white'}}>
-        You are using a beta version of NOMAD ({version.label}). {
+        You are using a {version.isBeta ? 'beta' : 'test'} version of NOMAD ({version.label}). {
           version.usesBetaData ? 'This version is not using the official data. Everything you upload here, might get lost.' : ''
-        } Click <a style={{color: 'white'}} href={version.officialUrl}>here for the official NOMAD version</a>.
+        } Click <MuiLink style={{color: 'white'}} href={version.officialUrl}>here for the official NOMAD version</MuiLink>.
       </span>}
       action={[
         <IconButton key={0} color="inherit" onClick={() => setUnderstood(true)}>
@@ -278,10 +279,16 @@ function MainMenu() {
       tooltip="Manage your data"
       icon={<UserDataIcon/>}
     />
+    {encyclopediaEnabled && <MainMenuItem
+      title="Encyclopedia"
+      href={`${appBase}/encyclopedia/#/search`}
+      tooltip="Visit the NOMAD Materials Encyclopedia"
+      icon={<EncyclopediaIcon/>}
+    />}
     {!oasis && aitoolkitEnabled && <MainMenuItem
-      title="Analytics"
-      path="/analytics"
-      tooltip="NOMAD's analytics (AI) toolkit tutorial jupyter notebooks"
+      title="AI Toolkit"
+      path="/aitoolkit"
+      tooltip="NOMAD's Artificial Intelligence Toolkit tutorial jupyter notebooks"
       icon={<AnalyticsIcon/>}
     />}
     <MainMenuItem
@@ -412,7 +419,8 @@ class NavigationUnstyled extends React.Component {
     '/userdata': 'Manage Your Data',
     '/metainfo': 'The NOMAD Meta Info',
     '/entry': 'Entry',
-    '/dataset': 'Dataset'
+    '/dataset': 'Dataset',
+    '/aitoolkit': 'Artificial Intelligence Toolkit'
   }
 
   toolbarHelp = {
@@ -477,9 +485,9 @@ class NavigationUnstyled extends React.Component {
                 disableGutters
               >
                 <div className={classes.title}>
-                  <a href="https://nomad-lab.eu">
+                  <MuiLink href="https://nomad-lab.eu">
                     <img alt="The NOMAD logo" className={classes.logo} src={`${guiBase}/nomad.png`}></img>
-                  </a>
+                  </MuiLink>
                   <Typography variant="h6" color="inherit" noWrap>
                     {selected(toolbarTitles)}
                   </Typography>
@@ -559,9 +567,9 @@ const routes = {
     path: '/metainfo',
     component: MetainfoPage
   },
-  'analytics': {
-    path: '/analytics',
-    component: AnalyticsPage
+  'aitoolkit': {
+    path: '/aitoolkit',
+    component: AIToolkitPage
   }
 }
 

gui/src/components/DataTable.js

@@ -14,9 +14,12 @@ import Checkbox from '@material-ui/core/Checkbox'
 import IconButton from '@material-ui/core/IconButton'
 import Tooltip from '@material-ui/core/Tooltip'
 import ViewColumnIcon from '@material-ui/icons/ViewColumn'
-import { Popover, List, ListItemText, ListItem, Collapse } from '@material-ui/core'
+import { Popover, List, ListItemText, ListItem, Collapse, Icon, Box } from '@material-ui/core'
 import { compose } from 'recompose'
 import _ from 'lodash'
+import { normalizeDisplayValue } from '../config'
+import SortIcon from '@material-ui/icons/Sort'
+import searchQuantities from '../searchQuantities'
 
 const globalSelectedColumns = {}
 
@@ -447,6 +450,7 @@ class DataTableUnStyled extends React.Component {
                 </TableCell> : <React.Fragment/>}
                 {Object.keys(columns).filter(key => selectedColumns.indexOf(key) !== -1).map(key => {
                   const column = columns[key]
+                  const description = column.description || (searchQuantities[key] && searchQuantities[key].description)
                   return (
                     <TableCell
                       key={key}
@@ -454,7 +458,7 @@ class DataTableUnStyled extends React.Component {
                       align={column.align || 'left'}
                       sortDirection={orderBy === key ? order : false}
                     >
-                      <Tooltip title={column.description || ''}>
+                      <Tooltip title={description || ''}>
                         {column.supportsSort ? <TableSortLabel
                           active={orderBy === key}