Merge branch 'v0.8.0' into encyclopedia

.gitignore

@@ -25,4 +25,4 @@ nomad.yaml
 ./gunicorn.conf
 build/
 dist/
-setup.json
\ No newline at end of file
+setup.json

Dockerfile

@@ -22,7 +22,19 @@
 # We use slim for the final image
 FROM python:3.6-slim as final
 
-# First, build all python stuff in a python build image
+# First built the GUI in a gui build image
+FROM node:latest as gui_build
+RUN mkdir -p /app
+WORKDIR /app
+ENV PATH /app/node_modules/.bin:$PATH
+COPY gui/package.json /app/package.json
+COPY gui/yarn.lock /app/yarn.lock
+RUN yarn
+COPY gui /app
+RUN yarn run build
+RUN yarn run --silent react-docgen src/components --pretty > react-docgen.out
+
+# Second, build all python stuff in a python build image
 FROM python:3.6-stretch as build
 RUN mkdir /install
 
@@ -56,23 +68,13 @@ RUN python setup.py compile
 RUN pip install .[all]
 RUN python setup.py sdist
 WORKDIR /install/docs
+COPY --from=gui_build /app/react-docgen.out /install/docs
 RUN make html
 RUN \
     find /usr/local/lib/python3.6/ -name 'tests' ! -path '*/networkx/*' -exec rm -r '{}' + && \
     find /usr/local/lib/python3.6/ -name 'test' -exec rm -r '{}' + && \
     find /usr/local/lib/python3.6/site-packages/ -name '*.so' -print -exec sh -c 'file "{}" | grep -q "not stripped" && strip -s "{}"' \;
 
-# Second built the GUI in a gui build image
-FROM node:latest as gui_build
-RUN mkdir -p /app
-WORKDIR /app
-ENV PATH /app/node_modules/.bin:$PATH
-COPY gui/package.json /app/package.json
-COPY gui/yarn.lock /app/yarn.lock
-RUN yarn
-COPY gui /app
-RUN yarn run build
-
 # Third, create a slim final image
 FROM final
 

nomad-meta-info @ c424e756

-Subproject commit 7a7c46ddeac23591b8d10aa6bbb7a7fa987df51c
+Subproject commit c424e75671e8c09c2f29c90ec63feafd0a2a706e

exciting @ 33098d6a

-Subproject commit a94784b1b8a3b5e780772e9ab573cfe3819ff7db
+Subproject commit 33098d6a162461dab7afdc8aa822ee9bb6be61bc

gaussian @ 3d51c6f6

-Subproject commit 2e2a4cc93fe2f2f91b8ce44f1da983f315cf1453
+Subproject commit 3d51c6f61e2c8b8eaa026dd613c23e9524c9e9ea

docs/.gitignore

 .build/
-*.graffle/
\ No newline at end of file
+*.graffle/
+react-docgen.out
\ No newline at end of file

docs/conf.py

@@ -13,8 +13,12 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 import os
 import sys
+sys.path.insert(0, os.path.abspath('.'))
 # from recommonmark.transform import AutoStructify
 
+# import docutils_react_docgen
+# docutils_react_docgen.SETTINGS['react_docgen'] = 'cat'
+
 sys.path.insert(0, os.path.abspath('..'))
 
 

docs/docutils_react_docgen.py

+# =====================
+# The MIT License (MIT)
+# =====================
+
+# Copyright (c) 2015 Paul Wexler
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+"""
+docutils_react_docgen
+=====================
+
+docutils extension for documenting React modules.
+Requires react-docgen
+"""
+
+from docutils import statemachine
+from docutils.parsers import rst
+import json
+import os
+import re
+import subprocess
+
+REACT_DOCGEN = 'react-docgen' # **Deprecated** Use SETTINGS['react_docgen']
+
+MODULE_DESCRIPTION_MISSING = 'Module doc string is missing!'
+
+MODULE_PROP_DESCRIPTION_MISSING = 'Property doc string is missing!'
+
+MODULE_UNDERLINE_CHARACTER = '-'
+
+TAB_SIZE = 4
+
+DEFAULT_OPTIONS = {
+    'exclude':                          '',
+    'include':                          '',
+    'module_description_missing':       MODULE_DESCRIPTION_MISSING,
+    'module_prop_description_missing':  MODULE_PROP_DESCRIPTION_MISSING,
+    'module_underline_character':       MODULE_UNDERLINE_CHARACTER,
+    'path_index':                       0,
+    'show_prop_type':                   False,
+    'src':                              '',
+    'tab_size':                         TAB_SIZE,
+    'use_commonjs_module_name':         True,
+}
+
+SETTINGS = {
+        'project_base': None,           # absolute path to project base.
+        'react_docgen': 'react-docgen', # react-docgen command to run.
+        'rst_output': None,             # output filename or no rst output.
+        }
+
+
+def find_package(dirname):
+    """find commonjs package for given directory.
+    Starts from `dirname` and recurses up the directory tree
+    looking for bower.json or package.json.
+
+    Returns a tuple (dirname, package)
+
+    dirname
+        The directory the .json file was found in.
+
+    package
+        A dict loaded from the .json file.
+        Its keys are the module filenames.
+    """
+    if dirname:
+        bower_json = os.path.join(dirname, 'bower.json')
+        if os.path.exists(bower_json):
+            with open(bower_json, 'r') as f:
+                return dirname, json.load(f)
+        package_json = os.path.join(dirname, 'package.json')
+        if os.path.exists(package_json):
+            with open(package_json, 'r') as f:
+                return dirname, json.load(f)
+        next_dirname = os.path.dirname(dirname)
+        if next_dirname != dirname:
+            return find_package(next_dirname)
+    return None, None
+
+
+def get_dirname(doc_dict, options):
+    return (os.path.dirname(list(doc_dict.keys())[0])
+            if options['use_commonjs_module_name'] and doc_dict
+            else '')
+
+def react_docgen(args, react_docgen=REACT_DOCGEN):
+    """Execute `react-docgen` with the given arguments.
+
+    `args` is a string which may contain spaces.
+
+    `react_docgen` is also a string which may contain spaces.
+
+    Returns the output of `react-docgen` as a dict
+    whose keys are module filenames (strings),
+    and whose values are module metadata (dicts).
+
+    WARNING
+        The default for react_docgen always evaluates to its initial value.
+    """
+    cmd = react_docgen.split() + args.split()
+    return json.loads(subprocess.check_output(cmd, stderr=subprocess.PIPE))
+
+def react_doc_to_rst(doc_dict, options, formatter_class, args=''):
+    """ Convert `doc_dict`, the react-docgen output dict
+    to a string of ReStructuredText,
+    according to the `options` and using the `formatter_class`
+
+    `args` is the string of arguments passed to the directive.
+    The default is ''.  Only required when using absolute addressing
+    so the Formatter can recover the path_argument.
+    """
+    dirname = get_dirname(doc_dict, options)
+    formatter = formatter_class(options, dirname, args=args)
+    return formatter.run(doc_dict)
+
+
+def run_react_docgen(args, options=DEFAULT_OPTIONS):
+    """ Execute `SETTINGS['react_docgen']` with the given args.
+
+    `args` is a string which may contain spaces.
+
+    `SETTINGS['react_docgen']` is also a string which may contain spaces.
+
+    `options` is a dict of directive options.
+
+    The command output is expected to be a JSON blob representing
+    a dict whose keys are the module filenames (strings),
+    and whose values are the module metadata (dicts).
+
+    However, the blob is simply converted into a python object and returned.
+
+    Implements the `project_base` setting and the `path_index` option.
+    """
+    arg_list = args.split()
+    project_base = SETTINGS['project_base']
+    if project_base != None:
+        path_index = options['path_index']
+        path_argument = arg_list[path_index]
+        if not path_argument.startswith(os.path.sep):
+            arg_list[path_index] = os.path.abspath(os.path.join(
+                    project_base,
+                    path_argument))
+    cmd = SETTINGS['react_docgen'].split() + arg_list
+    return json.loads(subprocess.check_output(cmd, stderr=subprocess.PIPE))
+
+class Formatter(object):
+    """ Formatter(options, dirname).run(doc_dict) returns a string.
+
+    options
+        a dict of options.
+
+    dirname
+        the directory to search for the CommonJS package
+        if the use_commonjs_module_name option is True
+
+    doc_dict
+        a dict of react-docgen module metadata
+
+    args
+        Default is ''.  The string of arguments passed to the directive.
+        Required when using absolute addressing.
+    """
+
+    def __init__(self, options, dirname, args=''):
+        self.options = options
+        self.tab = ' ' * self.options['tab_size']
+        package_dirname, package = find_package(dirname)
+        if package_dirname:
+            self.package_dirname_len = len(package_dirname)
+            self.package_name = package['name']
+        else:
+            self.package_dirname_len = 0
+        self.args = args
+        self._compile_filters()
+
+    def _compile_filters(self):
+        include = self.options['include']
+        self.include = re.compile(include) if include else None
+        exclude = self.options['exclude']
+        self.exclude = re.compile(exclude) if exclude else None
+
+    def _filter(self, filename, module_blob):
+        """returns True/False to include/exclude the given module
+        from the output.
+        """
+        description = module_blob.get('description', '')
+        return ((not self.include or self.include.search(description))
+                and
+                (not self.exclude or not self.exclude.search(description)))
+
+    def _get_module_name(self, filename):
+        if self.package_dirname_len:
+            module_name = '%s%s' % (
+                    self.package_name,
+                    filename[self.package_dirname_len:])
+            if module_name.endswith('.js'):
+                module_name = module_name[:-3]
+        else:
+            module_name = filename
+        return module_name
+
+    def _get_object_name(self, obj):
+        if 'value' in obj:
+            value = obj['value']
+            if isinstance(value, str):
+                return value
+            elif isinstance(value, list):
+                return '[%s]' % ', '.join(
+                        self._get_object_name(item) for item in value)
+            else:
+                return str(value)
+        elif 'name' in obj:
+            return obj['name']
+        else:
+            # if this happens show the obj instead of raising an error.
+            return str(obj)
+
+    def _make_definition(self, term, term_definition):
+        definition = '\n'.join((self.tab + line
+                for line in term_definition.split('\n'))
+                if term_definition else [self.tab])
+        return term + '\n' + definition
+
+    def _make_emphasis(self, text, style):
+        s = ''
+        s += style + text + style
+        return s
+
+    def _make_heading(self, text, underline_char):
+        s = ''
+        s += text + '\n'
+        s += underline_char * len(text) + '\n\n'
+        return s
+
+    def _make_module(self, filename, module_blob):
+        s = ''
+        s += self._make_module_header(filename)
+        s += self._make_module_description(module_blob)
+        s += self._make_module_props(module_blob)
+        return s
+
+    def _make_module_description(self, module_blob):
+        s = ''
+        description = module_blob.get('description', '')
+        s += description if description else self.options[
+                'module_description_missing']
+        s += '\n\n'
+        return s
+
+    def _make_module_header(self, filename):
+        module_name = self._get_module_name(filename)
+        s = ''
+        s += self._make_heading(
+                module_name,
+                self.options['module_underline_character'])
+        s += self._make_src_link(filename)
+        return s
+
+    def _make_module_prop_name(self, name, prop):
+        args = []
+        if self.options['show_prop_type'] and prop.get('type'):
+            args.append(self._get_object_name(prop['type']))
+        if prop.get('required'):
+            args.append('required')
+        if prop.get('defaultValue'):
+            args.append('default = ``%s``' % prop['defaultValue']['value'])
+        if args:
+            return '%s (%s)' % (name, ', '.join(args))
+        else:
+            return name
+
+    def _make_module_prop(self, name, prop):
+        return self._make_definition(
+                self._make_module_prop_name(name, prop),
+                self._make_module_prop_description(prop)) + '\n\n'
+
+    def _make_module_prop_description(self, prop):
+        return prop.get(
+                'description',
+                self.options['module_prop_description_missing'])
+
+    def _make_module_props(self, module_blob):
+        s = ''
+        props = module_blob.get('props', {})
+        for key in sorted(props.keys()):
+            s += self._make_module_prop(key, props[key])
+        return s
+
+    def _make_src_link(self, filename):
+        s = ''
+        if self.options['src']:
+            if self.args:
+                path = self.args.split()[self.options['path_index']]
+                module_name = filename[filename.index(path):]
+            else:
+                module_name = filename
+            link = '%s/%s' % (
+                    self.options['src'],
+                    module_name)
+            s += '`%s`_' % module_name
+            s += '\n\n'
+            s += '.. _`%s`: %s' % (
+                    module_name,
+                    link)
+            s += '\n\n'
+        return s
+
+    def run(self, doc_dict):
+        return ''.join(self._make_module(k, d)
+                for k, d in sorted(doc_dict.items())
+                if self._filter(k, d))
+
+
+class ReactDocgen(rst.Directive):
+    """ Docutils Directive which calls the react-docgen executable.
+    """
+
+    required_arguments = 1
+    optional_arguments = 0
+    final_argument_whitespace = True
+    option_spec = {'src': rst.directives.unchanged}
+    option_spec.update(
+            {k.lower(): rst.directives.unchanged
+                    for k in DEFAULT_OPTIONS.keys()})
+    has_content = False
+    formatter_class = Formatter
+
+    def run(self):
+        args = self.arguments[0]
+        options = {}
+        options.update(DEFAULT_OPTIONS)
+        options.update(self.options)
+        doc_dict = run_react_docgen(args, options=options)
+        rst = react_doc_to_rst(
+                doc_dict,
+                options,
+                self.formatter_class,
+                args=args)
+        if SETTINGS['rst_output']:
+            with open(SETTINGS['rst_output'], 'w') as fo:
+                fo.write(rst)
+        tab_size = options['tab_size']
+        include_lines = statemachine.string2lines(
+                rst,
+                tab_size,
+                convert_whitespace=True)
+        self.state_machine.insert_input(include_lines, '')
+        return []
+
+rst.directives.register_directive('reactdocgen', ReactDocgen)
+

docs/gui.rst

+GUI React Components
+====================
+
+These is the API reference for NOMAD's GUI React components.
+
+.. contents:: Table of Contents
+.. reactdocgen:: react-docgen.out

docs/index.rst

@@ -16,4 +16,5 @@ and infrastructure with a simplyfied architecture and consolidated code base.
    parser_tutorial
    archive_tutorial
    reference
+   gui
    ops

examples/client.py

@@ -3,12 +3,12 @@ from nomad.client import query_archive
 from nomad.metainfo import units
 
 # this will not be necessary, once this is the official NOMAD version
-config.client.url = 'http://labdev-nomad.esc.rzg.mpg.de/fairdi/nomad/testing-major/api'
+config.client.url = 'https://labdev-nomad.esc.rzg.mpg.de/dev/nomad/v0-8-0/api'
 
 
 aq = query_archive(
     query={
-        'upload_id': ['b5rGMO6dT4Gzqn3JaLjPpw']
+        'upload_id': ['6LUBCju3T3KK3D_fRCJ4qw']
     },
     required={
         'section_run': {
@@ -23,6 +23,6 @@ print('total', aq.total)
 
 for i, e in enumerate(aq):
     if i % 200 == 0:
-        print(e.section_run[0].section_single_configuration_calculation[0].energy_total)
+        print(e.section_run[0].section_single_configuration_calculation[0].energy_total.to(units.hartree))
 
 print(aq)

gui/package.json

@@ -4,8 +4,9 @@
   "commit": "e98694e",
   "private": true,
   "dependencies": {
-    "@material-ui/core": "^3.9.3",
-    "@material-ui/icons": "^3.0.2",
+    "@material-ui/core": "^4.0.0",
+    "@material-ui/icons": "^4.0.0",
+    "@material-ui/lab": "^4.0.0-alpha.49",
     "@navjobs/upload": "^3.1.3",
     "autosuggest-highlight": "^3.1.1",
     "base-64": "^0.1.0",
@@ -19,30 +20,31 @@
     "lodash": "^4.17.15",
     "marked": "^0.6.0",
     "material-ui-chip-input": "^1.0.0-beta.14",
-    "material-ui-flat-pagination": "^3.2.0",
+    "material-ui-flat-pagination": "^4.0.0",
+    "object-hash": "^2.0.3",
     "pace": "^0.0.4",
     "pace-js": "^1.0.2",
     "piwik-react-router": "^0.12.1",
     "qs": "^6.8.0",
-    "react": "^16.4.2",
+    "react": "^16.8.0",
     "react-app-polyfill": "^1.0.1",
     "react-autosuggest": "^9.4.3",
     "react-cookie": "^3.0.8",
     "react-copy-to-clipboard": "^5.0.1",
-    "react-dom": "^16.4.2",
+    "react-dom": "^16.8.0",
     "react-dropzone": "^5.0.1",
     "react-highlight": "^0.12.0",
     "react-infinite-scroller": "^1.2.4",
     "react-json-view": "^1.19.1",
     "react-keycloak": "^6.1.0",
-    "react-router-dom": "^4.3.1",
-    "react-router-hash-link": "^1.2.0",
+    "react-router-dom": "^5.1.2",
     "react-scripts": "1.1.4",
     "react-swipeable-views": "^0.13.0",
     "recompose": "^0.28.2",
     "swagger-client": "^3.8.22",
     "three.js": "^0.77.1",
-    "url-parse": "^1.4.3"
+    "url-parse": "^1.4.3",
+    "use-query-params": "^0.6.0"
   },
   "scripts": {
     "generate-build-version": "node generateBuildVersion",
@@ -53,6 +55,7 @@
     "eject": "react-scripts eject"
   },
   "devDependencies": {
+    "@material-ui/codemod": "^4.5.0",
     "babel-eslint": "^8.2.6",
     "eslint": "^4.19.1",
     "eslint-config-standard": "^11.0.0",
@@ -61,6 +64,7 @@
     "eslint-plugin-promise": "^3.7.0",
     "eslint-plugin-react": "^7.11.1",
     "eslint-plugin-standard": "^3.1.0",
+    "react-docgen": "^5.3.0",
     "serve": "^10.0.0"
   },
   "homepage": "http://example.com/fairdi/nomad/latest/gui"

gui/src/components/About.js

@@ -11,14 +11,13 @@ import { domains } from './domains'
 class About extends React.Component {
   static propTypes = {
     classes: PropTypes.object.isRequired,
-    api: PropTypes.object.isRequired,
     info: PropTypes.object,
     raiseError: PropTypes.func.isRequired
   }
 
   static styles = theme => ({
     root: {
-      padding: theme.spacing.unit * 3
+      padding: theme.spacing(3)
     }
   })
 
@@ -30,10 +29,10 @@ class About extends React.Component {
         <Markdown>{`
           # The NOMAD Repository and Archive
 
-          This web-page is the graphical user interface (GUI) for the NOMAD Repository and
-          Archive. It allows you to search, access, and download all NOMAD data in its
-          raw (Repository) and processed (Archive) form. You can upload and manage your own
-          raw computational material science data. Learn more about what data can be uploaded
+          This is the *graphical user interface* (GUI) for the NOMAD Repository and
+          Archive. It allows you to **search, access, and download all NOMAD data** in its
+          raw (Repository) and processed (Archive) form. You can **upload and manage your own
+          raw materials science data**. Learn more about what data can be uploaded
           and how to prepare your data on the [NOMAD Repository homepage](https://repository.nomad-coe.eu/).
           You can access all published data without an account. If you want to provide
           your own data, please login or register for an account.
@@ -42,17 +41,6 @@ class About extends React.Component {
           components as an effort to consolidate the various web applications from the
           NOMAD Repository, Archive, Metainfo, Encyclopedia, and Analytics Toolkit.
 
-          ### This looks different, what about the old NOMAD interface?
-
-          We have migrated all data from the original NOMAD Repository to this new system.
-          However, not all of the data was successfully processed by the new and more powerful parsers.
-          We will continue to improve the parsers to raise the quality of archive data overtime.
-          For some entries, no archive data might be currently available and some metadata might
-          still be missing when you are exploring Nomad data using the new search and data
-          exploring capabilities (menu items on the left).
-
-          ### Terms of use and licenses
-          ${consent}
 
           ### Getting Help
           If you encounter any difficulties, please write to
@@ -61,14 +49,8 @@ class About extends React.Component {
           about possible features, feel free to open an issue on our [issue tracking
           system](https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-FAIR/issues).
 
-          ### Material science data and domains
-          Originally NOMAD was build for DFT calculations and data from the respective
-          community code. By NOMAD supports multiple materials science domains:
-
-          ${info && info.domains.map(domain => domains[domain.name]).map(domain => `- ${domain.name}: ${domain.about}`).join('\n')}
-
-          ### ReST APIs