Added nexus metainfo generator. Adapted metainfo browser to new metainfo features. #672

.gitmodules

@@ -213,3 +213,6 @@
 [submodule "dependencies/parsers/asr"]
 	path = dependencies/parsers/asr
 	url = https://github.com/nomad-coe/nomad-parser-asr.git
+[submodule "dependencies/nexus_definitions"]
+	path = dependencies/nexus_definitions
+	url = https://github.com/nexusformat/definitions

nexus_definitions @ 493adbea

+Subproject commit 493adbeaa6a66f24b15c0882ca4c6594b0692f5f

gui/src/components/archive/Browser.js

@@ -143,6 +143,7 @@ const useLaneStyles = makeStyles(theme => ({
     display: 'inline-block'
   },
   container: {
+    minWidth: 300,
     display: 'inline-block',
     height: '100%',
     overflowY: 'scroll'
@@ -280,7 +281,7 @@ List.propTypes = ({
 })
 
 export function Content({children}) {
-  return <Box padding={1} paddingRight={2} maxWidth={1024}>
+  return <Box padding={1} maxWidth={1024}>
     {children}
   </Box>
 }
@@ -296,7 +297,7 @@ export function Compartment({title, children, color}) {
     return ''
   }
   return <React.Fragment>
-    <Box paddingTop={1}>
+    <Box paddingTop={1} whiteSpace="nowrap">
       {title && <Typography color={color} variant="overline">{title}</Typography>}
     </Box>
     {children}

gui/src/components/archive/MetainfoBrowser.js

@@ -20,7 +20,7 @@ import PropTypes from 'prop-types'
 import { useRecoilValue, useRecoilState, atom } from 'recoil'
 import { configState } from './ArchiveBrowser'
 import Browser, { Item, Content, Compartment, Adaptor, laneContext, formatSubSectionName } from './Browser'
-import { Typography, Box, makeStyles, Grid, FormGroup, TextField, Button } from '@material-ui/core'
+import { Typography, Box, makeStyles, Grid, FormGroup, TextField, Button, Tooltip } from '@material-ui/core'
 import { metainfoDef, resolveRef, vicinityGraph, rootSections, path as metainfoPath, packagePrefixes, defsByName } from './metainfo'
 import * as d3 from 'd3'
 import blue from '@material-ui/core/colors/blue'
@@ -154,7 +154,7 @@ export function metainfoAdaptorFactory(obj) {
   if (obj.m_def === 'Section') {
     return new SectionDefAdaptor(obj)
   } else if (obj.m_def === 'SubSection') {
-    return new Error('SubSections are not represented in the browser')
+    return new SubSectionDefAdaptor(obj)
   } else if (obj.m_def === 'Quantity') {
     return new QuantityDefAdaptor(obj)
   } else if (obj.m_def === 'Category') {
@@ -277,21 +277,42 @@ export class SectionDefAdaptor extends MetainfoAdaptor {
     if (key === '_baseSection') {
       return metainfoAdaptorFactory(resolveRef(this.e.base_sections[0]))
     }
-    const property = this.e._properties[key]
-    if (!property) {
-      return super.itemAdaptor(key)
-    } else {
-      if (property.m_def === 'SubSection') {
-        return metainfoAdaptorFactory(resolveRef(property.sub_section))
+
+    if (key.includes('@')) {
+      const [type, name] = key.split('@')
+      if (type === 'innerSectionDef') {
+        const innerSectionDef = this.e.inner_section_definitions.find(def => def.name === name)
+        if (innerSectionDef) {
+          return metainfoAdaptorFactory(innerSectionDef)
+        }
       }
+    }
+
+    const property = this.e._properties[key]
+    if (property) {
       return metainfoAdaptorFactory(property)
     }
+
+    return super.itemAdaptor(key)
   }
   render() {
     return <SectionDef def={this.e} />
   }
 }
 
+class SubSectionDefAdaptor extends MetainfoAdaptor {
+  constructor(e) {
+    super(e)
+    this.sectionDefAdaptor = new SectionDefAdaptor(resolveRef(e.sub_section))
+  }
+  itemAdaptor(key) {
+    return this.sectionDefAdaptor.itemAdaptor(key)
+  }
+  render() {
+    return <SubSectionDef def={this.e} />
+  }
+}
+
 class QuantityDefAdaptor extends MetainfoAdaptor {
   render() {
     return <QuantityDef def={this.e} />
@@ -314,6 +335,9 @@ function SectionDef({def}) {
     if (def._package.name.startsWith('nomad')) {
       return true
     }
+    if (def._package.name.startsWith('nexus')) {
+      return true
+    }
     if (metainfoConfig.packagePrefix) {
       return def._package.name.startsWith(metainfoConfig.packagePrefix)
     }
@@ -322,9 +346,11 @@ function SectionDef({def}) {
     }
     return false
   }
+
   return <Content style={{backgroundColor: 'grey'}}>
     <Definition def={def} kindLabel="section definition" />
-    {def.extends_base_section &&
+    <DefinitionProperties def={def} />
+    {def.base_sections.length > 0 &&
       <Compartment title="base section">
         {def.base_sections.map(baseSectionRef => {
           const baseSection = resolveRef(baseSectionRef)
@@ -334,13 +360,6 @@ function SectionDef({def}) {
         })}
       </Compartment>
     }
-    <Compartment title="squalified name">
-      <Typography>
-        <Box fontWeight="bold" component="span">
-          {def._qualifiedName}
-        </Box>
-      </Typography>
-    </Compartment>
     <Compartment title="sub section definitions">
       {def.sub_sections.filter(filter)
         .map(subSectionDef => {
@@ -375,6 +394,24 @@ function SectionDef({def}) {
         })
       }
     </Compartment>
+    <Compartment title="inner section definitions">
+      {def.inner_section_definitions.filter(filter)
+        .map(innerSectionDef => {
+          const key = `innerSectionDef@${innerSectionDef.name}`
+          const categories = innerSectionDef.categories?.map(c => resolveRef(c))
+          const unused = categories?.find(c => c.name === 'Unused')
+          return <Item key={key} itemKey={key}>
+            <Box component="span" whiteSpace="nowrap">
+              <Typography component="span" color={unused && 'error'}>
+                <Box fontWeight="bold" component="span">
+                  {innerSectionDef.name}
+                </Box>
+              </Typography>
+            </Box>
+          </Item>
+        })
+      }
+    </Compartment>
     <DefinitionDetails def={def} />
   </Content>
 }
@@ -382,14 +419,48 @@ SectionDef.propTypes = ({
   def: PropTypes.object
 })
 
+function SubSectionDef({def}) {
+  return <React.Fragment>
+    <Content>
+      <Definition def={def} kindLabel="sub section definition"/>
+      <DefinitionProperties def={def}>
+        {def.repeats && <Typography><b>repeats</b>:&nbsp;true</Typography>}
+      </DefinitionProperties>
+    </Content>
+    <SectionDef def={resolveRef(def.sub_section)} />
+  </React.Fragment>
+}
+SubSectionDef.propTypes = ({
+  def: PropTypes.object
+})
+
+function DefinitionProperties({def, children}) {
+  if (!(children || def.aliases?.length || def.deprecated || Object.keys(def.more).length)) {
+    return ''
+  }
+
+  return <Compartment title="properties">
+    {children}
+    {def.aliases?.length && <Typography><b>aliases</b>:&nbsp;{def.aliases.map(a => `"${a}"`).join(', ')}</Typography>}
+    {def.deprecated && <Typography><b>deprecated</b>: {def.deprecated}</Typography>}
+    {Object.keys(def.more).map((moreKey, i) => (
+      <Typography key={i}><b>{moreKey}</b>:&nbsp;{def.more[moreKey]}</Typography>
+    ))}
+  </Compartment>
+}
+DefinitionProperties.propTypes = ({
+  def: PropTypes.object,
+  children: PropTypes.any
+})
+
 function QuantityDef({def}) {
   return <Content>
     <Definition def={def} kindLabel="quantity definition"/>
-    <Compartment title="properties">
+    <DefinitionProperties def={def}>
       {def.type.type_kind !== 'reference'
         ? <Typography>
           <b>type</b>:&nbsp;
-          {def.type.type_data}&nbsp;
+          {Array.isArray(def.type.type_data) ? def.type.type_data.join(', ') : def.type.type_data}&nbsp;
           {def.type.type_kind !== 'data' && `(${def.type.type_kind})`}
         </Typography>
         : <Item itemKey="_reference">
@@ -403,10 +474,8 @@ function QuantityDef({def}) {
       </Typography>
       {def.unit &&
         <Typography><b>unit</b>:&nbsp;{def.unit}</Typography>}
-      {def.aliases && def.aliases !== [] && <Typography><b>aliases</b>:&nbsp;{def.aliases.map(a => `"${a}"`).join(', ')}</Typography>}
       {def.derived && <Typography><b>derived</b></Typography>}
-    </Compartment>
-    <DefinitionDetails def={def} />
+    </DefinitionProperties>
   </Content>
 }
 QuantityDef.propTypes = ({
@@ -419,7 +488,9 @@ function Definition({def, ...props}) {
     {def.description && !def.extends_base_section &&
       <Compartment title="description">
         <Box marginTop={1} marginBottom={1}>
-          <Markdown>{def.description}</Markdown>
+          {def._qualifiedName.startsWith('nexus')
+            ? <Typography>{def.description}</Typography>
+            : <Markdown>{def.description}</Markdown>}
         </Box>
       </Compartment>
     }
@@ -521,7 +592,9 @@ export function Title({def, isDefinition, data, kindLabel}) {
   return <Compartment>
     <Grid container justifyContent="space-between" wrap="nowrap" spacing={1}>
       <Grid item>
-        <Typography color={color} variant="h6">{def.name}</Typography>
+        <Tooltip title={def._qualifiedName || def.name}>
+          <Typography color={color} variant="h6">{def.name}</Typography>
+        </Tooltip>
         <DefinitionLabel def={def} isDefinition={isDefinition} variant="caption" color={color} />
       </Grid>
       <Grid item>

gui/src/components/archive/metainfo.js

@@ -29,6 +29,7 @@ const addDef = def => {
     defsForName.push(def)
   }
   defs.push(def)
+  def.more = def.more || {}
 }
 
 function sortDefs(defs) {
@@ -45,20 +46,25 @@ metainfo.packages.forEach(pkg => {
   pkg.category_definitions = pkg.category_definitions || []
   pkg.section_definitions = pkg.section_definitions || []
   pkg.category_definitions.forEach(categoryDef => {
-    categoryDef._qualifiedName = `${pkg.name}:${categoryDef.name}`
+    categoryDef._qualifiedName = `${pkg.name}.${categoryDef.name}`
     categoryDef._package = pkg
     addDef(categoryDef)
   })
-  pkg.section_definitions.forEach(sectionDef => {
+
+  const addSectionDef = (sectionDef, parentDef) => {
     pkg._sections[sectionDef.name] = sectionDef
+    sectionDef.base_sections = sectionDef.base_sections || []
     sectionDef.quantities = sectionDef.quantities || []
     sectionDef.sub_sections = sectionDef.sub_sections || []
+    sectionDef.inner_section_definitions = sectionDef.inner_section_definitions || []
     sectionDef._incomingRefs = sectionDef._incomingRefs || []
     sectionDef._parentSections = sectionDef._parentSections || []
     sectionDef._parentSubSections = sectionDef._parentSubSections || []
-    sectionDef._qualifiedName = `${pkg.name}:${sectionDef.name}`
+    sectionDef._qualifiedName = parentDef ? `${parentDef._qualifiedName || parentDef.name}.${sectionDef.name}` : sectionDef.name
     sectionDef._package = pkg
 
+    sectionDef.inner_section_definitions.forEach(innerSectionDef => addSectionDef(innerSectionDef, sectionDef))
+
     const addPropertiesFromSections = sections => sections
       .map(ref => resolveRef(ref)).forEach(extendingSectionDef => {
         if (extendingSectionDef.quantities) {
@@ -71,8 +77,6 @@ metainfo.packages.forEach(pkg => {
     sectionDef.extending_sections = sectionDef.extending_sections || []
     addPropertiesFromSections(sectionDef.extending_sections)
     if (!sectionDef.extends_base_section) {
-      addPropertiesFromSections(sectionDef.base_sections)
-      sectionDef.base_sections = sectionDef.base_sections || []
       addDef(sectionDef)
     }
 
@@ -81,7 +85,7 @@ metainfo.packages.forEach(pkg => {
       sectionDef._properties[property.name] = property
       if (!sectionDef.extends_base_section) {
         property._section = sectionDef
-        property._qualifiedName = `${sectionDef._qualifiedName}:${property.name}`
+        property._qualifiedName = `${sectionDef._qualifiedName}.${property.name}`
       }
       property._parentSections = [sectionDef]
       property._package = pkg
@@ -105,7 +109,9 @@ metainfo.packages.forEach(pkg => {
       subSectionsSectionDef._parentSubSections.push(subSection)
       subSection._section = sectionDef
     })
-  })
+  }
+
+  pkg.section_definitions.forEach(sectionDef => addSectionDef(sectionDef, pkg))
 })
 
 export const rootSections = sortDefs(defs.filter(def => (

nomad/cli/dev.py

@@ -77,6 +77,9 @@ def metainfo_undecorated():
     from nomad.parsing import parsers
     parsers.parsers
 
+    # TODO this is otherwise not imported and will add nexus to the Package.registry
+    from nomad.datamodel.metainfo import nexus  # pylint: disable=unused-import
+
     # TODO we call __init_metainfo__() for all packages where this has been forgotten
     # by the package author. Ideally this would not be necessary and we fix the
     # actual package definitions.

nomad/datamodel/metainfo/nexus.py

+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from typing import Dict, Any
+import xml.etree.ElementTree as ET
+import os.path
+import os
+import sys
+import numpy as np
+import re
+
+from nomad.utils import strip
+from nomad.metainfo import (
+    Section, Package, SubSection, Definition, Datetime, Bytes, MEnum, Quantity)
+from nomad.datamodel import EntryArchive
+
+
+# url_regexp from https://stackoverflow.com/questions/3809401/what-is-a-good-regular-expression-to-match-a-url
+url_regexp = re.compile(r'(https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*))')
+xml_namespaces = {'nx': 'http://definition.nexusformat.org/nxdl/3.1'}
+
+# TODO the validation still show some problems. Most notably there are a few higher
+# dimensional fields with non number types, which the metainfo does not support
+validate = False
+current_package: Package = None
+
+_definition_sections: Dict[str, Section] = dict()
+
+
+def to_camel_case(snake_str: str, upper: bool = False) -> str:
+    components = snake_str.split('_')
+    if upper:
+        return ''.join(f'{x[0].upper()}{x[1:]}' for x in components)
+
+    return components[0] + ''.join(f'{x[0].upper()}{x[1:]}' for x in components[1:])
+
+
+def get_section(name: str, **kwargs) -> Section:
+    '''
+    Returns the 'existing' metainfo section for a given top-level nexus base-class name.
+
+    This function ensures that sections for these base-classes are only created one.
+    This allows to access the metainfo section even before it is generated from the base-class
+    nexus definition.
+    '''
+    if name in _definition_sections:
+        section = _definition_sections[name]
+        section.more.update(**kwargs)
+        return section
+
+    section = Section(validate=validate, name=name, more=kwargs)
+    current_package.section_definitions.append(section)
+    _definition_sections[section.name] = section
+
+    return section
+
+
+def add_definition_properties(xml_node: ET.Element, definition: Definition, name_prefix: str = None):
+    '''
+    Adds general metainfo definition properties (e.g. name, deprecated, description)
+    from the given nexus XML node to the given metainfo definition.
+    '''
+    xml_attrs = xml_node.attrib
+
+    if 'name' in xml_attrs:
+        name = xml_attrs['name']
+        if name_prefix:
+            name = f'{name_prefix}_{name}'
+        definition.name = name
+
+    doc = xml_node.find('nx:doc', xml_namespaces)
+    if doc is not None and doc.text is not None:
+        definition.description = strip(doc.text)
+        urls = []
+        for match in url_regexp.findall(definition.description):
+            urls.append(match[0])
+        if len(urls) > 0:
+            definition.links = urls
+
+    if 'deprecated' in xml_attrs:
+        definition.deprecated = xml_attrs['deprecated']
+
+
+def get_enum(xml_node: ET.Element):
+    enumeration = xml_node.find('nx:enumeration', xml_namespaces)
+    if enumeration is not None:
+        enum_values = []
+        for enum_value in enumeration.findall('nx:item', xml_namespaces):
+            enum_values.append(enum_value.attrib['value'])
+        return MEnum(*enum_values)
+    return None
+
+
+def add_attributes(xml_node: ET.Element, section: Section):
+    '''
+    Adds quantities for all attributes in the given nexus XML node to the given
+    section.
+    '''
+    for attribute in xml_node.findall('nx:attribute', xml_namespaces):
+        type: Any = get_enum(xml_node)
+        if type is None:
+            type = str
+        quantity = Quantity(type=type, nx_kind='attribute')
+        add_definition_properties(attribute, quantity)
+        section.quantities.append(quantity)
+
+
+# TODO There are more types in nxdl, but they are not used by the current base classes and
+# application definitions.
+_nx_types = {
+    'NX_FLOAT': np.dtype(np.float64),
+    'NX_CHAR': str,
+    'NX_BOOLEAN': bool,
+    'NX_INT': np.dtype(np.int64),
+    'NX_NUMBER': np.dtype(np.number),
+    'NX_POSINT': np.dtype(np.uint64),
+    'NX_BINARY': Bytes,
+    'NX_DATE_TIME': Datetime
+}
+
+
+def section_from_field(xml_node: ET.Element) -> Section:
+    '''
+    Generates a metainfo section for the given nexus field XML node.
+    '''
+    xml_attrs = xml_node.attrib
+
+    name = to_camel_case(xml_attrs['name'], True) + 'Field'
+    section = Section(validate=validate, name=name, more=dict(nx_kind='field'))
+    value = Quantity(name='field_value')
+    section.quantities.append(value)
+
+    if 'type' in xml_attrs:
+        nx_type = xml_attrs['type']
+        if nx_type not in _nx_types:
+            raise NotImplementedError(f'type {nx_type} is not supported')
+        value.type = _nx_types[nx_type]
+    else:
+        value.type = get_enum(xml_node)
+        if value.type is None:
+            value.type = Any
+
+    if 'unit' in xml_attrs:
+        # TODO map unit
+        pass
+
+    dimensions = xml_node.find('nx:dimensions', xml_namespaces)
+    if dimensions is not None:
+        shape = []
+        for dimension in dimensions.findall('nx:dim', xml_namespaces):
+            dimension_value: Any = dimension.attrib.get('value', '*')
+            try:
+                dimension_value = int(dimension_value)
+            except ValueError:
+                pass
+            shape.append(dimension_value)
+        value.shape = shape
+
+    add_attributes(xml_node, section)
+
+    return section
+
+
+def add_group_properties(xml_node: ET.Element, definition_section: Section):
+    '''
+    Adds all properties that can be generated from the given nexus group XML node to
+    the given (empty) metainfo section definition.
+    '''
+    add_attributes(xml_node, definition_section)
+
+    for group in xml_node.findall('nx:group', xml_namespaces):
+        assert 'type' in group.attrib, 'group has not type'
+
+        type = group.attrib['type']
+        base_section = get_section(type)
+        empty_definition = len(group) == 0 or (
+            len(group) == 1 and group.find('nx:doc', xml_namespaces) is not None)
+        if empty_definition:
+            # The group does not define anything new, we can directly use the base definition
+            group_section = base_section
+        else:
+            if 'name' in group.attrib:
+                name = to_camel_case(group.attrib['name'], True) + 'Group'
+            else:
+                name = to_camel_case(type, True) + 'Group'
+
+            group_section = Section(validate=validate, name=name)
+            group_section.base_sections = [base_section]
+            definition_section.inner_section_definitions.append(group_section)
+            add_group_properties(group, group_section)
+
+        sub_section = SubSection(section_def=group_section, nx_kind='group')
+        add_definition_properties(group, sub_section, name_prefix='nx_group')
+        if sub_section.name is None:
+            sub_section.name = type.replace('NX', 'nx_group_')
+        definition_section.sub_sections.append(sub_section)
+
+    for field in xml_node.findall('nx:field', xml_namespaces):
+        assert 'name' in field.attrib, 'field has not name'
+
+        field_section = section_from_field(field)
+        definition_section.inner_section_definitions.append(field_section)
+
+        more = dict(nx_kind='field')
+        more.update(**{
+            f'nx_{key}': field.attrib[key]
+            for key in ['type', 'units'] if key in field.attrib})
+        field_sub_section = SubSection(section_def=field_section, **more)
+        add_definition_properties(field, field_sub_section, name_prefix='nx_field')
+        definition_section.sub_sections.append(field_sub_section)
+
+    return definition_section
+
+
+def section_from_definition(definition_file: str):
+    '''
+    Creates a metainfo section from the top-level nexus group definition in the given
+    nxdl file and adds it to the current metainfo package.
+    '''
+    xml_tree = ET.parse(definition_file)
+    definition = xml_tree.getroot()
+    xml_attrs = definition.attrib
+
+    assert xml_attrs.get('type') == 'group', 'definition is not a group'
+    assert 'name' in xml_attrs
+
+    definition_section = get_section(xml_attrs['name'], nx_kind=xml_attrs['type'])
+    if 'extends' in xml_attrs:
+        base_section = get_section(xml_attrs['extends'])
+        definition_section.base_sections = [base_section]
+    add_group_properties(definition, definition_section)
+    add_definition_properties(definition, definition_section)
+
+
+def package_from_directory(path: str, package_name: str) -> Package:
+    '''
+    Creates a metainfo package from the given nexus directory. Will the respective
+    metainfo definitions generated from all the nxdl files in that directory.
+    '''
+    global current_package
+    current_package = Package(name=package_name)
+
+    for definition_file in sorted(os.listdir(path)):
+        if not definition_file.endswith('.nxdl.xml'):
+            continue
+
+        try:
+            section_from_definition(
+                os.path.join(path, definition_file))
+        except Exception as e:
+            print(f'Exception while mapping {definition_file}', file=sys.stderr)
+            raise e