Commit 484a7076 authored by Lauri Himanen's avatar Lauri Himanen
Browse files

Merge branch 'v0.8.0' into encyclopedia

parents 5443c908 c59d0526
Pipeline #74066 passed with stages
in 25 minutes and 19 seconds
......@@ -22,7 +22,19 @@
# We use slim for the final image
FROM python:3.6-slim as final
# First, build all python stuff in a python build image
# First built the GUI in a gui build image
FROM node:latest as gui_build
RUN mkdir -p /app
WORKDIR /app
ENV PATH /app/node_modules/.bin:$PATH
COPY gui/package.json /app/package.json
COPY gui/yarn.lock /app/yarn.lock
RUN yarn
COPY gui /app
RUN yarn run build
RUN yarn run --silent react-docgen src/components --pretty > react-docgen.out
# Second, build all python stuff in a python build image
FROM python:3.6-stretch as build
RUN mkdir /install
......@@ -56,23 +68,13 @@ RUN python setup.py compile
RUN pip install .[all]
RUN python setup.py sdist
WORKDIR /install/docs
COPY --from=gui_build /app/react-docgen.out /install/docs
RUN make html
RUN \
find /usr/local/lib/python3.6/ -name 'tests' ! -path '*/networkx/*' -exec rm -r '{}' + && \
find /usr/local/lib/python3.6/ -name 'test' -exec rm -r '{}' + && \
find /usr/local/lib/python3.6/site-packages/ -name '*.so' -print -exec sh -c 'file "{}" | grep -q "not stripped" && strip -s "{}"' \;
# Second built the GUI in a gui build image
FROM node:latest as gui_build
RUN mkdir -p /app
WORKDIR /app
ENV PATH /app/node_modules/.bin:$PATH
COPY gui/package.json /app/package.json
COPY gui/yarn.lock /app/yarn.lock
RUN yarn
COPY gui /app
RUN yarn run build
# Third, create a slim final image
FROM final
......
Subproject commit 7a7c46ddeac23591b8d10aa6bbb7a7fa987df51c
Subproject commit c424e75671e8c09c2f29c90ec63feafd0a2a706e
Subproject commit a94784b1b8a3b5e780772e9ab573cfe3819ff7db
Subproject commit 33098d6a162461dab7afdc8aa822ee9bb6be61bc
Subproject commit 2e2a4cc93fe2f2f91b8ce44f1da983f315cf1453
Subproject commit 3d51c6f61e2c8b8eaa026dd613c23e9524c9e9ea
.build/
*.graffle/
react-docgen.out
\ No newline at end of file
......@@ -13,8 +13,12 @@
# documentation root, use os.path.abspath to make it absolute, like shown here.
import os
import sys
sys.path.insert(0, os.path.abspath('.'))
# from recommonmark.transform import AutoStructify
# import docutils_react_docgen
# docutils_react_docgen.SETTINGS['react_docgen'] = 'cat'
sys.path.insert(0, os.path.abspath('..'))
......
# =====================
# The MIT License (MIT)
# =====================
# Copyright (c) 2015 Paul Wexler
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
"""
docutils_react_docgen
=====================
docutils extension for documenting React modules.
Requires react-docgen
"""
from docutils import statemachine
from docutils.parsers import rst
import json
import os
import re
import subprocess
REACT_DOCGEN = 'react-docgen' # **Deprecated** Use SETTINGS['react_docgen']
MODULE_DESCRIPTION_MISSING = 'Module doc string is missing!'
MODULE_PROP_DESCRIPTION_MISSING = 'Property doc string is missing!'
MODULE_UNDERLINE_CHARACTER = '-'
TAB_SIZE = 4
DEFAULT_OPTIONS = {
'exclude': '',
'include': '',
'module_description_missing': MODULE_DESCRIPTION_MISSING,
'module_prop_description_missing': MODULE_PROP_DESCRIPTION_MISSING,
'module_underline_character': MODULE_UNDERLINE_CHARACTER,
'path_index': 0,
'show_prop_type': False,
'src': '',
'tab_size': TAB_SIZE,
'use_commonjs_module_name': True,
}
SETTINGS = {
'project_base': None, # absolute path to project base.
'react_docgen': 'react-docgen', # react-docgen command to run.
'rst_output': None, # output filename or no rst output.
}
def find_package(dirname):
"""find commonjs package for given directory.
Starts from `dirname` and recurses up the directory tree
looking for bower.json or package.json.
Returns a tuple (dirname, package)
dirname
The directory the .json file was found in.
package
A dict loaded from the .json file.
Its keys are the module filenames.
"""
if dirname:
bower_json = os.path.join(dirname, 'bower.json')
if os.path.exists(bower_json):
with open(bower_json, 'r') as f:
return dirname, json.load(f)
package_json = os.path.join(dirname, 'package.json')
if os.path.exists(package_json):
with open(package_json, 'r') as f:
return dirname, json.load(f)
next_dirname = os.path.dirname(dirname)
if next_dirname != dirname:
return find_package(next_dirname)
return None, None
def get_dirname(doc_dict, options):
return (os.path.dirname(list(doc_dict.keys())[0])
if options['use_commonjs_module_name'] and doc_dict
else '')
def react_docgen(args, react_docgen=REACT_DOCGEN):
"""Execute `react-docgen` with the given arguments.
`args` is a string which may contain spaces.
`react_docgen` is also a string which may contain spaces.
Returns the output of `react-docgen` as a dict
whose keys are module filenames (strings),
and whose values are module metadata (dicts).
WARNING
The default for react_docgen always evaluates to its initial value.
"""
cmd = react_docgen.split() + args.split()
return json.loads(subprocess.check_output(cmd, stderr=subprocess.PIPE))
def react_doc_to_rst(doc_dict, options, formatter_class, args=''):
""" Convert `doc_dict`, the react-docgen output dict
to a string of ReStructuredText,
according to the `options` and using the `formatter_class`
`args` is the string of arguments passed to the directive.
The default is ''. Only required when using absolute addressing
so the Formatter can recover the path_argument.
"""
dirname = get_dirname(doc_dict, options)
formatter = formatter_class(options, dirname, args=args)
return formatter.run(doc_dict)
def run_react_docgen(args, options=DEFAULT_OPTIONS):
""" Execute `SETTINGS['react_docgen']` with the given args.
`args` is a string which may contain spaces.
`SETTINGS['react_docgen']` is also a string which may contain spaces.
`options` is a dict of directive options.
The command output is expected to be a JSON blob representing
a dict whose keys are the module filenames (strings),
and whose values are the module metadata (dicts).
However, the blob is simply converted into a python object and returned.
Implements the `project_base` setting and the `path_index` option.
"""
arg_list = args.split()
project_base = SETTINGS['project_base']
if project_base != None:
path_index = options['path_index']
path_argument = arg_list[path_index]
if not path_argument.startswith(os.path.sep):
arg_list[path_index] = os.path.abspath(os.path.join(
project_base,
path_argument))
cmd = SETTINGS['react_docgen'].split() + arg_list
return json.loads(subprocess.check_output(cmd, stderr=subprocess.PIPE))
class Formatter(object):
""" Formatter(options, dirname).run(doc_dict) returns a string.
options
a dict of options.
dirname
the directory to search for the CommonJS package
if the use_commonjs_module_name option is True
doc_dict
a dict of react-docgen module metadata
args
Default is ''. The string of arguments passed to the directive.
Required when using absolute addressing.
"""
def __init__(self, options, dirname, args=''):
self.options = options
self.tab = ' ' * self.options['tab_size']
package_dirname, package = find_package(dirname)
if package_dirname:
self.package_dirname_len = len(package_dirname)
self.package_name = package['name']
else:
self.package_dirname_len = 0
self.args = args
self._compile_filters()
def _compile_filters(self):
include = self.options['include']
self.include = re.compile(include) if include else None
exclude = self.options['exclude']
self.exclude = re.compile(exclude) if exclude else None
def _filter(self, filename, module_blob):
"""returns True/False to include/exclude the given module
from the output.
"""
description = module_blob.get('description', '')
return ((not self.include or self.include.search(description))
and
(not self.exclude or not self.exclude.search(description)))
def _get_module_name(self, filename):
if self.package_dirname_len:
module_name = '%s%s' % (
self.package_name,
filename[self.package_dirname_len:])
if module_name.endswith('.js'):
module_name = module_name[:-3]
else:
module_name = filename
return module_name
def _get_object_name(self, obj):
if 'value' in obj:
value = obj['value']
if isinstance(value, str):
return value
elif isinstance(value, list):
return '[%s]' % ', '.join(
self._get_object_name(item) for item in value)
else:
return str(value)
elif 'name' in obj:
return obj['name']
else:
# if this happens show the obj instead of raising an error.
return str(obj)
def _make_definition(self, term, term_definition):
definition = '\n'.join((self.tab + line
for line in term_definition.split('\n'))
if term_definition else [self.tab])
return term + '\n' + definition
def _make_emphasis(self, text, style):
s = ''
s += style + text + style
return s
def _make_heading(self, text, underline_char):
s = ''
s += text + '\n'
s += underline_char * len(text) + '\n\n'
return s
def _make_module(self, filename, module_blob):
s = ''
s += self._make_module_header(filename)
s += self._make_module_description(module_blob)
s += self._make_module_props(module_blob)
return s
def _make_module_description(self, module_blob):
s = ''
description = module_blob.get('description', '')
s += description if description else self.options[
'module_description_missing']
s += '\n\n'
return s
def _make_module_header(self, filename):
module_name = self._get_module_name(filename)
s = ''
s += self._make_heading(
module_name,
self.options['module_underline_character'])
s += self._make_src_link(filename)
return s
def _make_module_prop_name(self, name, prop):
args = []
if self.options['show_prop_type'] and prop.get('type'):
args.append(self._get_object_name(prop['type']))
if prop.get('required'):
args.append('required')
if prop.get('defaultValue'):
args.append('default = ``%s``' % prop['defaultValue']['value'])
if args:
return '%s (%s)' % (name, ', '.join(args))
else:
return name
def _make_module_prop(self, name, prop):
return self._make_definition(
self._make_module_prop_name(name, prop),
self._make_module_prop_description(prop)) + '\n\n'
def _make_module_prop_description(self, prop):
return prop.get(
'description',
self.options['module_prop_description_missing'])
def _make_module_props(self, module_blob):
s = ''
props = module_blob.get('props', {})
for key in sorted(props.keys()):
s += self._make_module_prop(key, props[key])
return s
def _make_src_link(self, filename):
s = ''
if self.options['src']:
if self.args:
path = self.args.split()[self.options['path_index']]
module_name = filename[filename.index(path):]
else:
module_name = filename
link = '%s/%s' % (
self.options['src'],
module_name)
s += '`%s`_' % module_name
s += '\n\n'
s += '.. _`%s`: %s' % (
module_name,
link)
s += '\n\n'
return s
def run(self, doc_dict):
return ''.join(self._make_module(k, d)
for k, d in sorted(doc_dict.items())
if self._filter(k, d))
class ReactDocgen(rst.Directive):
""" Docutils Directive which calls the react-docgen executable.
"""
required_arguments = 1
optional_arguments = 0
final_argument_whitespace = True
option_spec = {'src': rst.directives.unchanged}
option_spec.update(
{k.lower(): rst.directives.unchanged
for k in DEFAULT_OPTIONS.keys()})
has_content = False
formatter_class = Formatter
def run(self):
args = self.arguments[0]
options = {}
options.update(DEFAULT_OPTIONS)
options.update(self.options)
doc_dict = run_react_docgen(args, options=options)
rst = react_doc_to_rst(
doc_dict,
options,
self.formatter_class,
args=args)
if SETTINGS['rst_output']:
with open(SETTINGS['rst_output'], 'w') as fo:
fo.write(rst)
tab_size = options['tab_size']
include_lines = statemachine.string2lines(
rst,
tab_size,
convert_whitespace=True)
self.state_machine.insert_input(include_lines, '')
return []
rst.directives.register_directive('reactdocgen', ReactDocgen)
GUI React Components
====================
These is the API reference for NOMAD's GUI React components.
.. contents:: Table of Contents
.. reactdocgen:: react-docgen.out
......@@ -16,4 +16,5 @@ and infrastructure with a simplyfied architecture and consolidated code base.
parser_tutorial
archive_tutorial
reference
gui
ops
......@@ -3,12 +3,12 @@ from nomad.client import query_archive
from nomad.metainfo import units
# this will not be necessary, once this is the official NOMAD version
config.client.url = 'http://labdev-nomad.esc.rzg.mpg.de/fairdi/nomad/testing-major/api'
config.client.url = 'https://labdev-nomad.esc.rzg.mpg.de/dev/nomad/v0-8-0/api'
aq = query_archive(
query={
'upload_id': ['b5rGMO6dT4Gzqn3JaLjPpw']
'upload_id': ['6LUBCju3T3KK3D_fRCJ4qw']
},
required={
'section_run': {
......@@ -23,6 +23,6 @@ print('total', aq.total)
for i, e in enumerate(aq):
if i % 200 == 0:
print(e.section_run[0].section_single_configuration_calculation[0].energy_total)
print(e.section_run[0].section_single_configuration_calculation[0].energy_total.to(units.hartree))
print(aq)
......@@ -4,8 +4,9 @@
"commit": "e98694e",
"private": true,
"dependencies": {
"@material-ui/core": "^3.9.3",
"@material-ui/icons": "^3.0.2",
"@material-ui/core": "^4.0.0",
"@material-ui/icons": "^4.0.0",
"@material-ui/lab": "^4.0.0-alpha.49",
"@navjobs/upload": "^3.1.3",
"autosuggest-highlight": "^3.1.1",
"base-64": "^0.1.0",
......@@ -19,30 +20,31 @@
"lodash": "^4.17.15",
"marked": "^0.6.0",
"material-ui-chip-input": "^1.0.0-beta.14",
"material-ui-flat-pagination": "^3.2.0",
"material-ui-flat-pagination": "^4.0.0",
"object-hash": "^2.0.3",
"pace": "^0.0.4",
"pace-js": "^1.0.2",
"piwik-react-router": "^0.12.1",
"qs": "^6.8.0",
"react": "^16.4.2",
"react": "^16.8.0",
"react-app-polyfill": "^1.0.1",
"react-autosuggest": "^9.4.3",
"react-cookie": "^3.0.8",
"react-copy-to-clipboard": "^5.0.1",
"react-dom": "^16.4.2",
"react-dom": "^16.8.0",
"react-dropzone": "^5.0.1",
"react-highlight": "^0.12.0",
"react-infinite-scroller": "^1.2.4",
"react-json-view": "^1.19.1",
"react-keycloak": "^6.1.0",
"react-router-dom": "^4.3.1",
"react-router-hash-link": "^1.2.0",
"react-router-dom": "^5.1.2",
"react-scripts": "1.1.4",
"react-swipeable-views": "^0.13.0",
"recompose": "^0.28.2",
"swagger-client": "^3.8.22",
"three.js": "^0.77.1",
"url-parse": "^1.4.3"
"url-parse": "^1.4.3",
"use-query-params": "^0.6.0"
},
"scripts": {
"generate-build-version": "node generateBuildVersion",
......@@ -53,6 +55,7 @@
"eject": "react-scripts eject"
},
"devDependencies": {
"@material-ui/codemod": "^4.5.0",
"babel-eslint": "^8.2.6",
"eslint": "^4.19.1",
"eslint-config-standard": "^11.0.0",
......@@ -61,6 +64,7 @@
"eslint-plugin-promise": "^3.7.0",
"eslint-plugin-react": "^7.11.1",
"eslint-plugin-standard": "^3.1.0",
"react-docgen": "^5.3.0",
"serve": "^10.0.0"
},
"homepage": "http://example.com/fairdi/nomad/latest/gui"
......
......@@ -11,14 +11,13 @@ import { domains } from './domains'
class About extends React.Component {
static propTypes = {
classes: PropTypes.object.isRequired,
api: PropTypes.object.isRequired,
info: PropTypes.object,
raiseError: PropTypes.func.isRequired
}
static styles = theme => ({
root: {
padding: theme.spacing.unit * 3
padding: theme.spacing(3)
}
})
......@@ -30,10 +29,10 @@ class About extends React.Component {
<Markdown>{`
# The NOMAD Repository and Archive
This web-page is the graphical user interface (GUI) for the NOMAD Repository and
Archive. It allows you to search, access, and download all NOMAD data in its
raw (Repository) and processed (Archive) form. You can upload and manage your own
raw computational material science data. Learn more about what data can be uploaded
This is the *graphical user interface* (GUI) for the NOMAD Repository and
Archive. It allows you to **search, access, and download all NOMAD data** in its
raw (Repository) and processed (Archive) form. You can **upload and manage your own
raw materials science data**. Learn more about what data can be uploaded
and how to prepare your data on the [NOMAD Repository homepage](https://repository.nomad-coe.eu/).
You can access all published data without an account. If you want to provide
your own data, please login or register for an account.
......@@ -42,17 +41,6 @@ class About extends React.Component {
components as an effort to consolidate the various web applications from the
NOMAD Repository, Archive, Metainfo, Encyclopedia, and Analytics Toolkit.
### This looks different, what about the old NOMAD interface?
We have migrated all data from the original NOMAD Repository to this new system.
However, not all of the data was successfully processed by the new and more powerful parsers.
We will continue to improve the parsers to raise the quality of archive data overtime.
For some entries, no archive data might be currently available and some metadata might
still be missing when you are exploring Nomad data using the new search and data
exploring capabilities (menu items on the left).
### Terms of use and licenses
${consent}
### Getting Help
If you encounter any difficulties, please write to
......@@ -61,14 +49,8 @@ class About extends React.Component {
about possible features, feel free to open an issue on our [issue tracking
system](https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-FAIR/issues).
### Material science data and domains
Originally NOMAD was build for DFT calculations and data from the respective
community code. By NOMAD supports multiple materials science domains:
${info && info.domains.map(domain => domains[domain.name]).map(domain => `- ${domain.name}: ${domain.about}`).join('\n')}
### ReST APIs
NOMAD services can also be accessed programmatically via ReST APIs.
### APIs
The NOMAD Repository and Archive can also be accessed programmatically via ReST APIs.
There is the proprietary NOMAD API and an implementation of the
[OPTiMaDe API (0.10.0)](https://github.com/Materials-Consortia/OPTiMaDe/tree/master)