Commit 10a4b215 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added nexus metainfo generator. Adapted metainfo browser to new metainfo features. #672

parent 58da90ab
Pipeline #116607 failed with stages
in 36 minutes and 9 seconds
......@@ -213,3 +213,6 @@
[submodule "dependencies/parsers/asr"]
path = dependencies/parsers/asr
url = https://github.com/nomad-coe/nomad-parser-asr.git
[submodule "dependencies/nexus_definitions"]
path = dependencies/nexus_definitions
url = https://github.com/nexusformat/definitions
Subproject commit 493adbeaa6a66f24b15c0882ca4c6594b0692f5f
......@@ -143,6 +143,7 @@ const useLaneStyles = makeStyles(theme => ({
display: 'inline-block'
},
container: {
minWidth: 300,
display: 'inline-block',
height: '100%',
overflowY: 'scroll'
......@@ -280,7 +281,7 @@ List.propTypes = ({
})
export function Content({children}) {
return <Box padding={1} paddingRight={2} maxWidth={1024}>
return <Box padding={1} maxWidth={1024}>
{children}
</Box>
}
......@@ -296,7 +297,7 @@ export function Compartment({title, children, color}) {
return ''
}
return <React.Fragment>
<Box paddingTop={1}>
<Box paddingTop={1} whiteSpace="nowrap">
{title && <Typography color={color} variant="overline">{title}</Typography>}
</Box>
{children}
......
......@@ -20,7 +20,7 @@ import PropTypes from 'prop-types'
import { useRecoilValue, useRecoilState, atom } from 'recoil'
import { configState } from './ArchiveBrowser'
import Browser, { Item, Content, Compartment, Adaptor, laneContext, formatSubSectionName } from './Browser'
import { Typography, Box, makeStyles, Grid, FormGroup, TextField, Button } from '@material-ui/core'
import { Typography, Box, makeStyles, Grid, FormGroup, TextField, Button, Tooltip } from '@material-ui/core'
import { metainfoDef, resolveRef, vicinityGraph, rootSections, path as metainfoPath, packagePrefixes, defsByName } from './metainfo'
import * as d3 from 'd3'
import blue from '@material-ui/core/colors/blue'
......@@ -154,7 +154,7 @@ export function metainfoAdaptorFactory(obj) {
if (obj.m_def === 'Section') {
return new SectionDefAdaptor(obj)
} else if (obj.m_def === 'SubSection') {
return new Error('SubSections are not represented in the browser')
return new SubSectionDefAdaptor(obj)
} else if (obj.m_def === 'Quantity') {
return new QuantityDefAdaptor(obj)
} else if (obj.m_def === 'Category') {
......@@ -277,21 +277,42 @@ export class SectionDefAdaptor extends MetainfoAdaptor {
if (key === '_baseSection') {
return metainfoAdaptorFactory(resolveRef(this.e.base_sections[0]))
}
const property = this.e._properties[key]
if (!property) {
return super.itemAdaptor(key)
} else {
if (property.m_def === 'SubSection') {
return metainfoAdaptorFactory(resolveRef(property.sub_section))
if (key.includes('@')) {
const [type, name] = key.split('@')
if (type === 'innerSectionDef') {
const innerSectionDef = this.e.inner_section_definitions.find(def => def.name === name)
if (innerSectionDef) {
return metainfoAdaptorFactory(innerSectionDef)
}
}
}
const property = this.e._properties[key]
if (property) {
return metainfoAdaptorFactory(property)
}
return super.itemAdaptor(key)
}
render() {
return <SectionDef def={this.e} />
}
}
class SubSectionDefAdaptor extends MetainfoAdaptor {
constructor(e) {
super(e)
this.sectionDefAdaptor = new SectionDefAdaptor(resolveRef(e.sub_section))
}
itemAdaptor(key) {
return this.sectionDefAdaptor.itemAdaptor(key)
}
render() {
return <SubSectionDef def={this.e} />
}
}
class QuantityDefAdaptor extends MetainfoAdaptor {
render() {
return <QuantityDef def={this.e} />
......@@ -314,6 +335,9 @@ function SectionDef({def}) {
if (def._package.name.startsWith('nomad')) {
return true
}
if (def._package.name.startsWith('nexus')) {
return true
}
if (metainfoConfig.packagePrefix) {
return def._package.name.startsWith(metainfoConfig.packagePrefix)
}
......@@ -322,9 +346,11 @@ function SectionDef({def}) {
}
return false
}
return <Content style={{backgroundColor: 'grey'}}>
<Definition def={def} kindLabel="section definition" />
{def.extends_base_section &&
<DefinitionProperties def={def} />
{def.base_sections.length > 0 &&
<Compartment title="base section">
{def.base_sections.map(baseSectionRef => {
const baseSection = resolveRef(baseSectionRef)
......@@ -334,13 +360,6 @@ function SectionDef({def}) {
})}
</Compartment>
}
<Compartment title="squalified name">
<Typography>
<Box fontWeight="bold" component="span">
{def._qualifiedName}
</Box>
</Typography>
</Compartment>
<Compartment title="sub section definitions">
{def.sub_sections.filter(filter)
.map(subSectionDef => {
......@@ -375,6 +394,24 @@ function SectionDef({def}) {
})
}
</Compartment>
<Compartment title="inner section definitions">
{def.inner_section_definitions.filter(filter)
.map(innerSectionDef => {
const key = `innerSectionDef@${innerSectionDef.name}`
const categories = innerSectionDef.categories?.map(c => resolveRef(c))
const unused = categories?.find(c => c.name === 'Unused')
return <Item key={key} itemKey={key}>
<Box component="span" whiteSpace="nowrap">
<Typography component="span" color={unused && 'error'}>
<Box fontWeight="bold" component="span">
{innerSectionDef.name}
</Box>
</Typography>
</Box>
</Item>
})
}
</Compartment>
<DefinitionDetails def={def} />
</Content>
}
......@@ -382,14 +419,48 @@ SectionDef.propTypes = ({
def: PropTypes.object
})
function SubSectionDef({def}) {
return <React.Fragment>
<Content>
<Definition def={def} kindLabel="sub section definition"/>
<DefinitionProperties def={def}>
{def.repeats && <Typography><b>repeats</b>:&nbsp;true</Typography>}
</DefinitionProperties>
</Content>
<SectionDef def={resolveRef(def.sub_section)} />
</React.Fragment>
}
SubSectionDef.propTypes = ({
def: PropTypes.object
})
function DefinitionProperties({def, children}) {
if (!(children || def.aliases?.length || def.deprecated || Object.keys(def.more).length)) {
return ''
}
return <Compartment title="properties">
{children}
{def.aliases?.length && <Typography><b>aliases</b>:&nbsp;{def.aliases.map(a => `"${a}"`).join(', ')}</Typography>}
{def.deprecated && <Typography><b>deprecated</b>: {def.deprecated}</Typography>}
{Object.keys(def.more).map((moreKey, i) => (
<Typography key={i}><b>{moreKey}</b>:&nbsp;{def.more[moreKey]}</Typography>
))}
</Compartment>
}
DefinitionProperties.propTypes = ({
def: PropTypes.object,
children: PropTypes.any
})
function QuantityDef({def}) {
return <Content>
<Definition def={def} kindLabel="quantity definition"/>
<Compartment title="properties">
<DefinitionProperties def={def}>
{def.type.type_kind !== 'reference'
? <Typography>
<b>type</b>:&nbsp;
{def.type.type_data}&nbsp;
{Array.isArray(def.type.type_data) ? def.type.type_data.join(', ') : def.type.type_data}&nbsp;
{def.type.type_kind !== 'data' && `(${def.type.type_kind})`}
</Typography>
: <Item itemKey="_reference">
......@@ -403,10 +474,8 @@ function QuantityDef({def}) {
</Typography>
{def.unit &&
<Typography><b>unit</b>:&nbsp;{def.unit}</Typography>}
{def.aliases && def.aliases !== [] && <Typography><b>aliases</b>:&nbsp;{def.aliases.map(a => `"${a}"`).join(', ')}</Typography>}
{def.derived && <Typography><b>derived</b></Typography>}
</Compartment>
<DefinitionDetails def={def} />
</DefinitionProperties>
</Content>
}
QuantityDef.propTypes = ({
......@@ -419,7 +488,9 @@ function Definition({def, ...props}) {
{def.description && !def.extends_base_section &&
<Compartment title="description">
<Box marginTop={1} marginBottom={1}>
<Markdown>{def.description}</Markdown>
{def._qualifiedName.startsWith('nexus')
? <Typography>{def.description}</Typography>
: <Markdown>{def.description}</Markdown>}
</Box>
</Compartment>
}
......@@ -521,7 +592,9 @@ export function Title({def, isDefinition, data, kindLabel}) {
return <Compartment>
<Grid container justifyContent="space-between" wrap="nowrap" spacing={1}>
<Grid item>
<Typography color={color} variant="h6">{def.name}</Typography>
<Tooltip title={def._qualifiedName || def.name}>
<Typography color={color} variant="h6">{def.name}</Typography>
</Tooltip>
<DefinitionLabel def={def} isDefinition={isDefinition} variant="caption" color={color} />
</Grid>
<Grid item>
......
......@@ -29,6 +29,7 @@ const addDef = def => {
defsForName.push(def)
}
defs.push(def)
def.more = def.more || {}
}
function sortDefs(defs) {
......@@ -45,20 +46,25 @@ metainfo.packages.forEach(pkg => {
pkg.category_definitions = pkg.category_definitions || []
pkg.section_definitions = pkg.section_definitions || []
pkg.category_definitions.forEach(categoryDef => {
categoryDef._qualifiedName = `${pkg.name}:${categoryDef.name}`
categoryDef._qualifiedName = `${pkg.name}.${categoryDef.name}`
categoryDef._package = pkg
addDef(categoryDef)
})
pkg.section_definitions.forEach(sectionDef => {
const addSectionDef = (sectionDef, parentDef) => {
pkg._sections[sectionDef.name] = sectionDef
sectionDef.base_sections = sectionDef.base_sections || []
sectionDef.quantities = sectionDef.quantities || []
sectionDef.sub_sections = sectionDef.sub_sections || []
sectionDef.inner_section_definitions = sectionDef.inner_section_definitions || []
sectionDef._incomingRefs = sectionDef._incomingRefs || []
sectionDef._parentSections = sectionDef._parentSections || []
sectionDef._parentSubSections = sectionDef._parentSubSections || []
sectionDef._qualifiedName = `${pkg.name}:${sectionDef.name}`
sectionDef._qualifiedName = parentDef ? `${parentDef._qualifiedName || parentDef.name}.${sectionDef.name}` : sectionDef.name
sectionDef._package = pkg
sectionDef.inner_section_definitions.forEach(innerSectionDef => addSectionDef(innerSectionDef, sectionDef))
const addPropertiesFromSections = sections => sections
.map(ref => resolveRef(ref)).forEach(extendingSectionDef => {
if (extendingSectionDef.quantities) {
......@@ -71,8 +77,6 @@ metainfo.packages.forEach(pkg => {
sectionDef.extending_sections = sectionDef.extending_sections || []
addPropertiesFromSections(sectionDef.extending_sections)
if (!sectionDef.extends_base_section) {
addPropertiesFromSections(sectionDef.base_sections)
sectionDef.base_sections = sectionDef.base_sections || []
addDef(sectionDef)
}
......@@ -81,7 +85,7 @@ metainfo.packages.forEach(pkg => {
sectionDef._properties[property.name] = property
if (!sectionDef.extends_base_section) {
property._section = sectionDef
property._qualifiedName = `${sectionDef._qualifiedName}:${property.name}`
property._qualifiedName = `${sectionDef._qualifiedName}.${property.name}`
}
property._parentSections = [sectionDef]
property._package = pkg
......@@ -105,7 +109,9 @@ metainfo.packages.forEach(pkg => {
subSectionsSectionDef._parentSubSections.push(subSection)
subSection._section = sectionDef
})
})
}
pkg.section_definitions.forEach(sectionDef => addSectionDef(sectionDef, pkg))
})
export const rootSections = sortDefs(defs.filter(def => (
......
......@@ -77,6 +77,9 @@ def metainfo_undecorated():
from nomad.parsing import parsers
parsers.parsers
# TODO this is otherwise not imported and will add nexus to the Package.registry
from nomad.datamodel.metainfo import nexus # pylint: disable=unused-import
# TODO we call __init_metainfo__() for all packages where this has been forgotten
# by the package author. Ideally this would not be necessary and we fix the
# actual package definitions.
......
#
# Copyright The NOMAD Authors.
#
# This file is part of NOMAD. See https://nomad-lab.eu for further info.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from typing import Dict, Any
import xml.etree.ElementTree as ET
import os.path
import os
import sys
import numpy as np
import re
from nomad.utils import strip
from nomad.metainfo import (
Section, Package, SubSection, Definition, Datetime, Bytes, MEnum, Quantity)
from nomad.datamodel import EntryArchive
# url_regexp from https://stackoverflow.com/questions/3809401/what-is-a-good-regular-expression-to-match-a-url
url_regexp = re.compile(r'(https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*))')
xml_namespaces = {'nx': 'http://definition.nexusformat.org/nxdl/3.1'}
# TODO the validation still show some problems. Most notably there are a few higher
# dimensional fields with non number types, which the metainfo does not support
validate = False
current_package: Package = None
_definition_sections: Dict[str, Section] = dict()
def to_camel_case(snake_str: str, upper: bool = False) -> str:
components = snake_str.split('_')
if upper:
return ''.join(f'{x[0].upper()}{x[1:]}' for x in components)
return components[0] + ''.join(f'{x[0].upper()}{x[1:]}' for x in components[1:])
def get_section(name: str, **kwargs) -> Section:
'''
Returns the 'existing' metainfo section for a given top-level nexus base-class name.
This function ensures that sections for these base-classes are only created one.
This allows to access the metainfo section even before it is generated from the base-class
nexus definition.
'''
if name in _definition_sections:
section = _definition_sections[name]
section.more.update(**kwargs)
return section
section = Section(validate=validate, name=name, more=kwargs)
current_package.section_definitions.append(section)
_definition_sections[section.name] = section
return section
def add_definition_properties(xml_node: ET.Element, definition: Definition, name_prefix: str = None):
'''
Adds general metainfo definition properties (e.g. name, deprecated, description)
from the given nexus XML node to the given metainfo definition.
'''
xml_attrs = xml_node.attrib
if 'name' in xml_attrs:
name = xml_attrs['name']
if name_prefix:
name = f'{name_prefix}_{name}'
definition.name = name
doc = xml_node.find('nx:doc', xml_namespaces)
if doc is not None and doc.text is not None:
definition.description = strip(doc.text)
urls = []
for match in url_regexp.findall(definition.description):
urls.append(match[0])
if len(urls) > 0:
definition.links = urls
if 'deprecated' in xml_attrs:
definition.deprecated = xml_attrs['deprecated']
def get_enum(xml_node: ET.Element):
enumeration = xml_node.find('nx:enumeration', xml_namespaces)
if enumeration is not None:
enum_values = []
for enum_value in enumeration.findall('nx:item', xml_namespaces):
enum_values.append(enum_value.attrib['value'])
return MEnum(*enum_values)
return None
def add_attributes(xml_node: ET.Element, section: Section):
'''
Adds quantities for all attributes in the given nexus XML node to the given
section.
'''
for attribute in xml_node.findall('nx:attribute', xml_namespaces):
type: Any = get_enum(xml_node)
if type is None:
type = str
quantity = Quantity(type=type, nx_kind='attribute')
add_definition_properties(attribute, quantity)
section.quantities.append(quantity)
# TODO There are more types in nxdl, but they are not used by the current base classes and
# application definitions.
_nx_types = {
'NX_FLOAT': np.dtype(np.float64),
'NX_CHAR': str,
'NX_BOOLEAN': bool,
'NX_INT': np.dtype(np.int64),
'NX_NUMBER': np.dtype(np.number),
'NX_POSINT': np.dtype(np.uint64),
'NX_BINARY': Bytes,
'NX_DATE_TIME': Datetime
}
def section_from_field(xml_node: ET.Element) -> Section:
'''
Generates a metainfo section for the given nexus field XML node.
'''
xml_attrs = xml_node.attrib
name = to_camel_case(xml_attrs['name'], True) + 'Field'
section = Section(validate=validate, name=name, more=dict(nx_kind='field'))
value = Quantity(name='field_value')
section.quantities.append(value)
if 'type' in xml_attrs:
nx_type = xml_attrs['type']
if nx_type not in _nx_types:
raise NotImplementedError(f'type {nx_type} is not supported')
value.type = _nx_types[nx_type]
else:
value.type = get_enum(xml_node)
if value.type is None:
value.type = Any
if 'unit' in xml_attrs:
# TODO map unit
pass
dimensions = xml_node.find('nx:dimensions', xml_namespaces)
if dimensions is not None:
shape = []
for dimension in dimensions.findall('nx:dim', xml_namespaces):
dimension_value: Any = dimension.attrib.get('value', '*')
try:
dimension_value = int(dimension_value)
except ValueError:
pass
shape.append(dimension_value)
value.shape = shape
add_attributes(xml_node, section)
return section
def add_group_properties(xml_node: ET.Element, definition_section: Section):
'''
Adds all properties that can be generated from the given nexus group XML node to
the given (empty) metainfo section definition.
'''
add_attributes(xml_node, definition_section)
for group in xml_node.findall('nx:group', xml_namespaces):
assert 'type' in group.attrib, 'group has not type'
type = group.attrib['type']
base_section = get_section(type)
empty_definition = len(group) == 0 or (
len(group) == 1 and group.find('nx:doc', xml_namespaces) is not None)
if empty_definition:
# The group does not define anything new, we can directly use the base definition
group_section = base_section
else:
if 'name' in group.attrib:
name = to_camel_case(group.attrib['name'], True) + 'Group'
else:
name = to_camel_case(type, True) + 'Group'
group_section = Section(validate=validate, name=name)
group_section.base_sections = [base_section]
definition_section.inner_section_definitions.append(group_section)
add_group_properties(group, group_section)
sub_section = SubSection(section_def=group_section, nx_kind='group')
add_definition_properties(group, sub_section, name_prefix='nx_group')
if sub_section.name is None:
sub_section.name = type.replace('NX', 'nx_group_')
definition_section.sub_sections.append(sub_section)
for field in xml_node.findall('nx:field', xml_namespaces):
assert 'name' in field.attrib, 'field has not name'
field_section = section_from_field(field)
definition_section.inner_section_definitions.append(field_section)
more = dict(nx_kind='field')
more.update(**{
f'nx_{key}': field.attrib[key]
for key in ['type', 'units'] if key in field.attrib})
field_sub_section = SubSection(section_def=field_section, **more)
add_definition_properties(field, field_sub_section, name_prefix='nx_field')
definition_section.sub_sections.append(field_sub_section)
return definition_section
def section_from_definition(definition_file: str):
'''
Creates a metainfo section from the top-level nexus group definition in the given
nxdl file and adds it to the current metainfo package.
'''
xml_tree = ET.parse(definition_file)
definition = xml_tree.getroot()
xml_attrs = definition.attrib
assert xml_attrs.get('type') == 'group', 'definition is not a group'
assert 'name' in xml_attrs
definition_section = get_section(xml_attrs['name'], nx_kind=xml_attrs['type'])
if 'extends' in xml_attrs:
base_section = get_section(xml_attrs['extends'])
definition_section.base_sections = [base_section]
add_group_properties(definition, definition_section)
add_definition_properties(definition, definition_section)
def package_from_directory(path: str, package_name: str) -> Package:
'''
Creates a metainfo package from the given nexus directory. Will the respective
metainfo definitions generated from all the nxdl files in that directory.
'''
global current_package
current_package = Package(name=package_name)
for definition_file in sorted(os.listdir(path)):
if not definition_file.endswith('.nxdl.xml'):
continue
try:
section_from_definition(
os.path.join(path, definition_file))
except Exception as e:
print(f'Exception while mapping {definition_file}', file=sys.stderr)
raise e