Commit c00ba593 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added a new endpoint for the new metainfo and adapted the depending GUI.

parent 5753e4e5
Pipeline #74868 failed with stages
in 23 minutes and 19 seconds
......@@ -50,6 +50,7 @@ Omitted versions are plain bugfix releases with only minor changes and fixes.
- new underlying datamodel that allows to maintain multiple domains
- mulitple domains supported the GUI
- new metainfo implementation
- API endpoint to access the metainfo
- new archive based on new metainfo
- client library that serves archive data as objects (with tab completion) not dictionaries
- properties and user tab in the search GUI
......
......@@ -50,6 +50,7 @@ extensions = [
'sphinx.ext.ifconfig',
'sphinx.ext.napoleon',
'sphinx.ext.autosectionlabel',
'sphinx.ext.extlinks',
'sphinx_click.ext',
'sphinxcontrib.httpdomain',
'sphinxcontrib.autohttp.flask',
......@@ -197,3 +198,5 @@ def setup(app):
# 'enable_eval_rst': True
# }, True)
# app.add_transform(AutoStructify)
extlinks = {'api': ('https://repository.nomad-coe.eu/app/api/%s', 'NOMAD API ')}
\ No newline at end of file
......@@ -2,3 +2,51 @@ Metainfo
========
.. automodule:: nomad.metainfo
Accessing the Metainfo
----------------------
Above you learned what the metainfo is and how to create metainfo definitions and work
with metainfo data in Python. But how do you get access to the existing metainfo definitions
within NOMAD? We call the complete set of all metainfo definitions the *NOMAD Metainfo*.
This *NOMAD Metainfo* comprises definitions from various packages defined by all the
parsers and converters (and respective code outputs and formats) that NOMAD supports. In
addition there are *common* packages that contain definitions that might be relevant to
different kinds of archive data.
Python
______
In the NOMAD source-code all metainfo definitions are materialized as Python source files
that contain the definitions in the format described above. If you have installed the
NOMAD Python package (see :ref:`install-client`), you can simply import the respective
Python modules:
.. code-block:: python
from nomad.datamodel.metainfo.public import m_package
print(m_package.m_to_json(indent=2))
from nomad.datamodel.metainfo.public import section_run
my_run = section_run()
API
___
In addition, a JSON version of the NOMAD Metainfo is available through our API via the
``metainfo`` endpoint.
You can get :api:`one giant JSON with all definitions <metainfo/>`, or you
can access the metainfo for specific packages, e.g. the :api:`VASP metainfo <metainfo/vasp.json>`. The
returned JSON will also contain all packages that the requested package depends on.
Legacy metainfo version
_______________________
There are no metainfo files anymore. The old ``*.nomadmetainfo.json`` files are no
longer maintained, as the Python definitions in each parser/converter implementation are
now the normative artifact for the NOMAD Metainfo.
To get the NOMAD Metainfo in the format of the old NOMAD CoE project, you can use
the ``metainfo/legacy`` endpoint; e.g. the :api:`VASP legacy metainfo <metainfo/legacy/vasp.nomadmetainfo.json>`.
......@@ -533,7 +533,7 @@ class Api {
_metaInfoRepositories = {}
async getMetaInfo(pkg) {
pkg = pkg || 'all.nomadmetainfo.json'
pkg = pkg || 'common.nomadmetainfo.json'
const metaInfoRepository = this._metaInfoRepositories[pkg]
......@@ -544,12 +544,12 @@ class Api {
try {
const loadMetaInfo = async(path) => {
return this.swagger()
.then(client => client.apis.archive.get_metainfo({metainfo_package_name: path}))
.then(client => client.apis.metainfo.get_legacy_metainfo({metainfo_package_name: path}))
.catch(handleApiError)
.then(response => response.body)
}
const metaInfo = await loadMetaInfo(pkg)
const metaInfoRepository = new MetaInfoRepository(metaInfo)
const metaInfoRepository = new MetaInfoRepository({[pkg]: metaInfo})
this._metaInfoRepositories[pkg] = metaInfoRepository
return metaInfoRepository
......
......@@ -38,7 +38,7 @@ class MetainfoDialogUnstyled extends React.PureComponent {
handleGotoMetainfoBrowser() {
const {history, onClose, metaInfoData} = this.props
history.push(`/metainfo/${metaInfoData.name}`)
history.push(`/metainfo/${metaInfoData.package.name}/${metaInfoData.name}`)
onClose()
}
......
......@@ -94,7 +94,7 @@ class DefinitionCardUnstyled extends React.Component {
: ''
}
<CardButton position="center" size="medium" icon="launch"
component={props => <Link to={`/metainfo/${definition.name}`} {...props} />}
component={props => <Link to={`/metainfo/${definition.package.name}/${definition.name}`} {...props} />}
/>
<PopoverCardButton
position="center" icon="code" classes={{content: classes.source}}
......
import React, { Component } from 'react'
import { matchPath } from 'react-router-dom'
import React, { Component, useContext, useState, useEffect } from 'react'
import { matchPath, useLocation, useHistory, useRouteMatch } from 'react-router-dom'
import Viewer from './Viewer'
import PropTypes from 'prop-types'
import { withApi } from '../api'
import { withApi, apiContext } from '../api'
import MetainfoSearch from './MetainfoSearch'
import { FormControl, withStyles, Select, Input, MenuItem, ListItemText, InputLabel } from '@material-ui/core'
import { FormControl, withStyles, Select, Input, MenuItem, ListItemText, InputLabel, makeStyles } from '@material-ui/core'
import { compose } from 'recompose'
import { schema } from '../MetaInfoRepository'
import { errorContext } from '../errors'
export const help = `
The NOMAD *metainfo* defines all quantities used to represent archive data in
......@@ -21,11 +22,11 @@ The NOMAD metainfo contains three different *kinds* of definitions:
- **references**: References that allow to connect related sections.
All definitions have a name that you can search for. Furthermore, all definitions
are organized in packages. There is a *common* package with definitions that are
are organized in packages. There is a *common* pkg with definitions that are
used by all codes and there are packages for each code with code specific definitions.
You can select the package to browse below.
You can select the pkg to browse below.
Depending on the selected package, there are quite a large number of definitions.
Depending on the selected pkg, there are quite a large number of definitions.
You can use the *definition* field to search based on definition names.
All definitions are represented as *cards* below. Click on the various card items
......@@ -47,158 +48,100 @@ const MenuProps = {
}
}
class MetaInfoBrowser extends Component {
static propTypes = {
classes: PropTypes.object.isRequired,
api: PropTypes.object.isRequired,
loading: PropTypes.number,
raiseError: PropTypes.func.isRequired,
location: PropTypes.object.isRequired,
match: PropTypes.object.isRequired,
history: PropTypes.object.isRequired
const useStyles = makeStyles(theme => ({
root: {},
forms: {
padding: `${theme.spacing(3)}px ${theme.spacing(3)}px 0 ${theme.spacing(3)}px`
},
packageSelect: {
width: 300, height: 24
},
search: {
width: 450,
marginRight: theme.spacing(2)
}
}))
static styles = theme => ({
root: {},
forms: {
padding: `${theme.spacing(3)}px ${theme.spacing(3)}px 0 ${theme.spacing(3)}px`
},
packageSelect: {
width: 300
},
search: {
width: 450,
marginRight: theme.spacing(2)
}
})
initialState = {
domainRootSection: null,
metainfos: null,
allMetainfos: null,
selectedPackage: null,
loadedPackage: null
}
state = this.initialState
export default function MetaInfoBrowser(props) {
const classes = useStyles()
constructor(props) {
super(props)
this.handleSelectedPackageChanged = this.handleSelectedPackageChanged.bind(this)
this.handleSearch = this.handleSearch.bind(this)
}
const location = useLocation()
const history = useHistory()
metainfo() {
const { location } = this.props
const match = matchPath(location.pathname, {
path: `${this.props.match.path}/:metainfo?`
})
return match.params.metainfo
}
update(pkg) {
this.props.api.getInfo().then(info => {
const domain = info.domains.find(domain => domain.name === 'dft') // TODO deal with domains
this.props.api.getMetaInfo(pkg || domain.metainfo.all_package).then(metainfos => {
const metainfoName = this.metainfo() || domain.metainfo.root_section
const definition = metainfos.get(metainfoName)
if (!definition) {
this.props.history.push(`/metainfo/${domain.metainfo.root_section}`)
} else {
this.setState({loadedPackage: pkg, metainfos: metainfos})
}
}).catch(error => {
this.props.raiseError(error)
})
}).catch(error => {
this.props.raiseError(error)
})
}
init() {
this.props.api.getInfo().then(info => {
const domain = info.domains.find(domain => domain.name === 'dft') // TODO deal with domains
this.props.api.getMetaInfo(domain.metainfo.all_package).then(metainfos => {
const metainfoName = this.metainfo() || domain.metainfo.root_section
const definition = metainfos.get(metainfoName)
this.setState({
domainRootSection: domain.metainfo.root_section,
allMetainfos: metainfos,
selectedPackage: definition.package.name})
this.update(definition.package.name)
}).catch(error => {
this.props.raiseError(error)
})
}).catch(error => {
this.props.raiseError(error)
})
const match = matchPath(location.pathname, {
path: `${useRouteMatch().path}/:pkg?/:metainfo?`
})
const pkg = match.params.pkg || 'general'
const metainfoName = match.params.metainfo || 'section_run'
const {api, loading} = useContext(apiContext)
const {raiseError} = useContext(errorContext)
const [metainfos, setMetainfos] = useState(null)
const [packages, setPackages] = useState(null)
useEffect(() => {
api.getInfo().then(info => {
setPackages(info.metainfo_packages)
}).catch(raiseError)
}, [api])
useEffect(() => {
api.getMetaInfo(pkg).then(metainfos => {
const definition = metainfos.get(metainfoName)
if (!definition) {
history.push(`/metainfo/${pkg}/section_run`)
} else {
setMetainfos(metainfos)
}
}).catch(raiseError)
}, [pkg, metainfoName, api])
const handleSelectedPackageChanged = event => {
history.push(`/metainfo/${event.target.value}/section_run`)
}
componentDidUpdate(prevProps) {
if (prevProps.location.pathname !== this.props.location.pathname) {
this.setState(this.initialState)
this.init()
const handleSearch = metainfoName => {
if (metainfos.get(metainfoName)) {
history.push(`/metainfo/${pkg}/${metainfoName}`)
}
}
componentDidMount() {
this.init()
if (!metainfos || !packages) {
return <div />
}
handleSelectedPackageChanged(event) {
this.setState({selectedPackage: event.target.value})
this.update(event.target.value)
}
handleSearch(term) {
if (this.state.metainfos.get(term)) {
this.props.history.push(`/metainfo/${term}`)
}
}
render() {
const { classes, loading } = this.props
const { metainfos, selectedPackage, allMetainfos, loadedPackage, domainRootSection } = this.state
if (!metainfos || !allMetainfos) {
return <div />
}
const metainfoName = this.metainfo() || domainRootSection || 'section_run'
const metainfo = metainfos.resolve(metainfos.createProxy(metainfoName))
return <div>
<div className={classes.forms}>
<form style={{ display: 'flex' }}>
<MetainfoSearch
classes={{container: classes.search}}
suggestions={Object.values(metainfos.names).filter(metainfo => !schema.isPackage(metainfo))}
onChange={this.handleSearch}
/>
<FormControl disabled={loading > 0}>
<InputLabel htmlFor="select-multiple-checkbox">Package</InputLabel>
<Select
classes={{root: classes.packageSelect}}
value={selectedPackage}
onChange={this.handleSelectedPackageChanged}
input={<Input id="select-multiple-checkbox" />}
MenuProps={MenuProps}
>
{allMetainfos.contents
.map(pkg => pkg.name)
.map(name => {
return <MenuItem key={name} value={name}>
<ListItemText primary={name.substring(0, name.length - 19)} style={{margin: 0}} />
</MenuItem>
})
}
</Select>
</FormControl>
</form>
</div>
<Viewer key={loadedPackage} rootElement={metainfo} packages={metainfos.contents} />
const metainfo = metainfos.resolve(metainfos.createProxy(metainfoName))
console.log(metainfoName)
return <div>
<div className={classes.forms}>
<form style={{ display: 'flex' }}>
<MetainfoSearch
classes={{container: classes.search}}
suggestions={Object.values(metainfos.names).filter(metainfo => !schema.isPackage(metainfo))}
onChange={handleSearch}
/>
<FormControl disabled={loading > 0}>
<InputLabel htmlFor="select-multiple-checkbox">Package</InputLabel>
<Select
classes={{root: classes.packageSelect}}
value={pkg}
onChange={handleSelectedPackageChanged}
input={<Input id="select-multiple-checkbox" />}
MenuProps={MenuProps}
>
{packages
.map(name => {
return <MenuItem key={name} value={name}>
<ListItemText primary={name} style={{margin: 0}} />
</MenuItem>
})
}
</Select>
</FormControl>
</form>
</div>
}
<Viewer key={`${metainfo.package.name}/${metainfo.name}`} rootElement={metainfo} packages={metainfos.contents} />
</div>
}
export default compose(withApi(false), withStyles(MetaInfoBrowser.styles))(MetaInfoBrowser)
......@@ -60,6 +60,17 @@ class ViewerUnstyled extends React.Component {
}
}
componentDidUpdate(prevProps) {
if (this.props.rootElement !== prevProps.rootElement) {
this.setState({
definitions: [{
definition: this.props.rootElement,
state: []
}]
})
}
}
isVisible(definition) {
let state = { state: this.state.definitions }
while (state && state.definition !== definition) {
......
......@@ -25,4 +25,4 @@ There is a separate documentation for the API endpoints from a client perspectiv
'''
from .api import api, blueprint
from . import info, auth, upload, repo, archive, raw, mirror, dataset
from . import info, auth, upload, repo, archive, raw, mirror, dataset, metainfo
......@@ -19,16 +19,12 @@ The archive API of the nomad@FAIRDI APIs. This API is about serving processed
from typing import Dict, Any
from io import BytesIO
import os.path
from flask import request, g
from flask_restplus import abort, Resource, fields
import json
import orjson
import importlib
import urllib.parse
import metainfo
from nomad.files import UploadFiles, Restricted
from nomad.archive import query_archive, ArchiveQueryError
from nomad import search, config
......@@ -347,93 +343,3 @@ class ArchiveQueryResource(Resource):
results['results'] = data
return results, 200
@ns.route('/metainfo/<string:metainfo_package_name>')
@api.doc(params=dict(metainfo_package_name='The name of the metainfo package.'))
class MetainfoResource(Resource):
@api.doc('get_metainfo')
@api.response(404, 'The metainfo does not exist')
@api.response(200, 'Metainfo data send')
def get(self, metainfo_package_name):
'''
Get a metainfo definition file.
'''
try:
return load_metainfo(metainfo_package_name), 200
except FileNotFoundError:
parser_prefix = metainfo_package_name[:-len('.nomadmetainfo.json')]
try:
return load_metainfo(dict(
parser='%sparser' % parser_prefix,
path='%s.nomadmetainfo.json' % parser_prefix)), 200
except FileNotFoundError:
abort(404, message='The metainfo %s does not exist.' % metainfo_package_name)
metainfo_main_path = os.path.dirname(os.path.abspath(metainfo.__file__))
def load_metainfo(
package_name_or_dependency: str, dependency_source: str = None,
loaded_packages: Dict[str, Any] = None) -> Dict[str, Any]:
'''
Loads the given metainfo package and all its dependencies. Returns a dict with
all loaded package_names and respective packages.
Arguments:
package_name_or_dependency: The name of the package, or a nomadmetainfo dependency object.
dependency_source: The path of the metainfo that uses this function to load a relative dependency.
loaded_packages: Give a dict and the function will added freshly loaded packages
to it and return it.
'''
if loaded_packages is None:
loaded_packages = {}
if isinstance(package_name_or_dependency, str):
package_name = package_name_or_dependency
metainfo_path = os.path.join(metainfo_main_path, package_name)
else:
dependency = package_name_or_dependency
if 'relativePath' in dependency:
if dependency_source is None:
raise Exception(
'Can only load relative dependency from within another metainfo package')
metainfo_path = os.path.join(
os.path.dirname(dependency_source), dependency['relativePath'])
elif 'metainfoPath' in dependency:
metainfo_path = os.path.join(metainfo_main_path, dependency['metainfoPath'])
elif 'parser' in dependency:
parser = dependency['parser']
path = dependency['path']
try:
parser_module = importlib.import_module(parser).__file__
except Exception:
raise Exception('Parser not installed %s for metainfo path %s' % (parser, metainfo_path))
parser_directory = os.path.dirname(parser_module)
metainfo_path = os.path.join(parser_directory, path)
else:
raise Exception('Invalid dependency type in metainfo package %s' % metainfo_path)
package_name = os.path.basename(metainfo_path)
package_name = os.path.basename(package_name)
if package_name in loaded_packages:
return loaded_packages
with open(metainfo_path, 'rt') as f:
metainfo_json = json.load(f)
loaded_packages[package_name] = metainfo_json
for dependency in metainfo_json.get('dependencies', []):
load_metainfo(dependency, dependency_source=metainfo_path, loaded_packages=loaded_packages)
return loaded_packages
......@@ -56,6 +56,7 @@ statistics_info_model = api.model('StatisticsInfo', {
info_model = api.model('Info', {
'parsers': fields.List(fields.String),
'metainfo_packages': fields.List(fields.String),
'codes': fields.List(fields.String),
'normalizers': fields.List(fields.String),
'domains': fields.List(fields.Nested(model=domain_model)),
......@@ -96,6 +97,9 @@ class InfoResource(Resource):
'parsers': [
key[key.index('/') + 1:]
for key in parsing.parser_dict.keys()],
'metainfo_packages': ['general', 'general.experimental', 'common', 'public'] + sorted([
key[key.index('/') + 1:]
for key in parsing.parser_dict.keys()]),
'codes': sorted(set(codes), key=lambda x: x.lower()),
'normalizers': [normalizer.__name__ for normalizer in normalizing.normalizers],
'statistics': statistics(),
......
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
'''
The archive API of the nomad@FAIRDI APIs. This API is about serving processed
(parsed and normalized) calculation data in nomad's *meta-info* format.
'''
from flask_restplus import abort, Resource
import importlib
from nomad.metainfo.legacy import python_package_mapping, LegacyMetainfoEnvironment
from nomad.metainfo import Package
from nomad.parsing import parsers
from .api import api
ns = api.namespace(
'metainfo',
description='Access the NOMAD Metainfo (i.e. the archive\'s schema/definitions).')
@ns.route('/')
class AllMetainfoResource(Resource):
@api.doc('get_all_metainfo')
@api.response(200, 'Metainfo send')
def get(self):