Commit 32f9050b authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Refactored some aspects of ems domain. #403

parent 8f325955
Pipeline #81593 passed with stages
in 47 minutes and 14 seconds
......@@ -144,9 +144,6 @@
[submodule "dependencies/optimade-python-tools"]
path = dependencies/optimade-python-tools
url = https://github.com/markus1978/optimade-python-tools.git
[submodule "dependencies/parsers/eels"]
path = dependencies/parsers/eels
url = https://github.com/markus1978/eels.git
[submodule "dependencies/parsers/namd"]
path = dependencies/parsers/namd
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-namd.git
......
Subproject commit c6bf6c2ca01bff208e424b531b8c214ab709b9b8
Subproject commit b9b8f841709736cbfcb3676226a89c06498aa26f
Subproject commit b42d4aae3fe431ed4afbbc738f5e19cec12634bd
Subproject commit 0d67544ea06713b0cf333eeee8d6ef6929a47f5e
Subproject commit c4c75068f91f83fb9262eb09576710d11984db44
Subproject commit 0533ef8393da2501226d60f4db9c0d7be32dbf81
Subproject commit aa7fbc09aa3bde6b654b764d21cd62eaf2badc9e
......@@ -360,8 +360,8 @@ import json
from .metainfo import m_env
from nomad.parsing.parser import MatchingParser
from nomad.datamodel.metainfo.general_experimental import section_experiment as msection_experiment
from nomad.datamodel.metainfo.general_experimental import section_data as msection_data
from nomad.datamodel.metainfo.common_experimental import section_experiment as msection_experiment
from nomad.datamodel.metainfo.common_experimental import section_data as msection_data
from nomad.datamodel.metainfo.general_experimental_method import section_method as msection_method
from nomad.datamodel.metainfo.general_experimental_sample import section_sample as msection_sample
......
......@@ -9,6 +9,7 @@ import {
import EMSVisualizations from './ems/EMSVisualizations'
import QCMSEntryOverview from './qcms/QCMSEntryOverview'
import QCMSEntryCards from './qcms/QCMSEntryCards'
import { Link } from '@material-ui/core'
/* eslint-disable react/display-name */
......@@ -201,20 +202,32 @@ export const domains = ({
*/
searchResultColumns: {
'formula': {
label: 'Formula'
label: 'Formula',
supportsSort: true
},
'ems.chemical': {
label: 'Chemical',
supportsSort: true
},
'ems.method': {
label: 'Method'
label: 'Method',
supportsSort: true
},
'ems.data_type': {
label: 'Data',
supportsSort: true
},
'ems.experiment_location': {
label: 'Location'
'ems.origin_time': {
label: 'Date',
supportsSort: true,
render: entry => (entry.ems && entry.ems.origin_time && new Date(entry.ems.origin_time).toLocaleDateString()) || 'unavailable'
},
'ems.experiment_time': {
label: 'Date/Time',
render: entry => (entry.ems && entry.ems.experiment_time !== 'unavailable') ? new Date(entry.ems.experiment_time * 1000).toLocaleString() : 'unavailable'
'ems.repository_url': {
label: 'Source',
render: entry => <Link target="external" href={entry.ems.entry_repository_url}>{entry.ems.repository_url}</Link>
}
},
defaultSearchResultColumns: ['formula', 'ems.method', 'ems.experiment_location', 'ems.experiment_time'],
defaultSearchResultColumns: ['formula', 'ems.chemical', 'ems.method', 'ems.data_type', 'ems.origin_time', 'ems.repository_url'],
/**
* A component to render the domain specific quantities in the metadata card of
* the entry view. Needs to work with props: data (the entry data from the API),
......
......@@ -27,7 +27,7 @@ class EMSEntryCards extends React.Component {
<CardContent classes={{root: classes.cardContent}}>
<Markdown classes={{root: classes.description}}>{`
The data for this experiment is externally stored and managed. Download the raw experiment data:
[${data.ems.repository_url}](${data.ems.repository_url}).
[${data.ems && data.ems.repository_url}](${data.ems && data.ems.entry_repository_url}).
The meta data describing this experiment in its original format, can be
downloaded here directly:
......
import React from 'react'
import PropTypes from 'prop-types'
import Quantity from '../Quantity'
import { Typography } from '@material-ui/core'
import { Typography, Link } from '@material-ui/core'
import { apiBase } from '../../config'
export default class EMSEntryOverview extends React.Component {
......@@ -46,6 +46,15 @@ export default class EMSEntryOverview extends React.Component {
return (
<Quantity column>
<Quantity quantity="ems.experiment_summary" label="summary" {...this.props} />
{this.state.previewBroken
? data.ems.entry_repository_url && <Quantity label="preview" {...this.props}>
<Typography noWrap>
<a target="external" href={ems.entry_repository_url}>visit this entry on the external database</a>
</Typography>
</Quantity>
: <Quantity label="preview" {...this.props}>
<img alt="preview" style={{maxWidth: '100%', height: 'auto'}} src={relative_preview_url} onError={this.handleBrokenPreview}></img>
</Quantity>}
<Quantity row>
<Quantity column>
<Quantity row>
......@@ -56,26 +65,17 @@ export default class EMSEntryOverview extends React.Component {
</Quantity>
<Quantity quantity="ems.method" label="experimental method" noWrap {...this.props} />
<Quantity quantity="ems.experiment_location" label="experiment location" noWrap {...this.props} />
<Quantity label="experiment time" {...this.props}>
<Quantity label="experiment or experiment publish date" {...this.props}>
<Typography noWrap>{
data.ems.experiment_time && data.ems.experiment_time !== 'unavailable' ? new Date(data.ems.experiment_time * 1000).toLocaleString() : 'unavailable'
(ems && ems.origin_time && new Date(ems.origin_time).toLocaleDateString()) || 'unavailable'
}</Typography>
</Quantity>
<Quantity label="data" {...this.props}>
<Typography noWrap>
<a target="external" href={data.ems.repository_url}>{data.ems.repository_name}</a>
<Link target="external" href={ems.entry_repository_url}>{ems.repository_url}</Link>
</Typography>
</Quantity>
</Quantity>
{this.state.previewBroken
? data.ems.entry_repository_url && <Quantity label="preview" {...this.props}>
<Typography noWrap>
<a target="external" href={data.ems.entry_repository_url}>visit this entry on the external database</a>
</Typography>
</Quantity>
: <Quantity label="preview" {...this.props}>
<img alt="preview" style={{maxWidth: '100%', height: 'auto'}} src={relative_preview_url} onError={this.handleBrokenPreview}></img>
</Quantity>}
</Quantity>
</Quantity>
)
......
import React from 'react'
import React, { useContext, useState, useEffect } from 'react'
import PropTypes from 'prop-types'
import { withStyles, Divider, Card, CardContent, Grid, CardHeader, Typography, Link } from '@material-ui/core'
import { withApi } from '../api'
import { compose } from 'recompose'
import { Divider, Card, CardContent, Grid, CardHeader, Typography, Link, makeStyles } from '@material-ui/core'
import { apiContext } from '../api'
import ApiDialogButton from '../ApiDialogButton'
// import Structure from '../visualization/Structure'
import Quantity from '../Quantity'
......@@ -10,9 +9,9 @@ import { Link as RouterLink } from 'react-router-dom'
import { DOI } from '../search/DatasetList'
import { domains } from '../domains'
import { EntryPageContent } from './EntryPage'
import { errorContext } from '../errors'
class RepoEntryView extends React.Component {
static styles = theme => ({
const useStyles = makeStyles(theme => ({
root: {
marginTop: theme.spacing(2)
},
......@@ -29,55 +28,33 @@ class RepoEntryView extends React.Component {
height: '25rem',
padding: '0'
}
})
}))
static propTypes = {
classes: PropTypes.object.isRequired,
api: PropTypes.object.isRequired,
raiseError: PropTypes.func.isRequired,
uploadId: PropTypes.string.isRequired,
calcId: PropTypes.string.isRequired
}
export default function RepoEntryView({uploadId, calcId}) {
const classes = useStyles()
const {api} = useContext(apiContext)
const {raiseError} = useContext(errorContext)
const [state, setState] = useState({calcData: null, doesNotExist: false})
static defaultState = {
calcData: null,
doesNotExist: false
}
useEffect(() => {
setState({calcData: null, doesNotExist: false})
}, [setState, uploadId, calcId])
state = {...RepoEntryView.defaultState}
componentDidMount() {
this.update()
}
componentDidUpdate(prevProps) {
if (prevProps.api !== this.props.api ||
prevProps.uploadId !== this.props.uploadId ||
prevProps.calcId !== this.props.calcId) {
this.setState({...RepoEntryView.defaultState})
this.update()
}
}
update() {
const {uploadId, calcId} = this.props
this.props.api.repo(uploadId, calcId).then(data => {
this.setState({calcData: data, doesNotExist: false})
useEffect(() => {
api.repo(uploadId, calcId).then(data => {
setState({calcData: data, doesNotExist: false})
}).catch(error => {
this.setState({calcData: null, doesNotExist: false})
if (error.name === 'DoesNotExist') {
this.setState({doesNotExist: true})
setState({calcData: null, doesNotExist: true})
} else {
this.props.raiseError(error)
setState({calcData: null, doesNotExist: false})
raiseError(error)
}
})
}
}, [api, raiseError, uploadId, calcId, setState])
render() {
const { classes, ...calcProps } = this.props
const calcData = this.state.calcData || calcProps
const loading = !this.state.calcData
const { uploadId, calcId } = calcProps
const calcData = state.calcData || {uploadId: uploadId, calcId: calcId}
const loading = !state.calcData
const quantityProps = {data: calcData, loading: loading}
const authors = loading ? null : calcData.authors
......@@ -88,10 +65,12 @@ class RepoEntryView extends React.Component {
entryHeader = domain.entryTitle(calcData)
}
if (this.state.doesNotExist) {
return <Typography className={classes.error}>
if (state.doesNotExist) {
return <EntryPageContent className={classes.root} fixed>
<Typography className={classes.error}>
This entry does not exist.
</Typography>
</EntryPageContent>
}
return (
......@@ -173,7 +152,9 @@ class RepoEntryView extends React.Component {
{domain && <domain.EntryCards data={calcData} calcId={calcId} uploadId={uploadId} classes={{root: classes.entryCards}} />}
</EntryPageContent>
)
}
}
export default compose(withApi(false, true), withStyles(RepoEntryView.styles))(RepoEntryView)
RepoEntryView.propTypes = {
uploadId: PropTypes.string.isRequired,
calcId: PropTypes.string.isRequired
}
......@@ -24,7 +24,7 @@ def dev():
pass
@dev.command(help='Runs tests and linting of the nomad source code. Useful before committing code.')
@dev.command(help='Runs tests and linting of the nomad python source code. Useful before committing code.')
@click.option('--skip-tests', help='Do no tests, just do code checks.', is_flag=True)
@click.option('-x', '--exitfirst', help='Stop testing after first failed test case.', is_flag=True)
def qa(skip_tests: bool, exitfirst: bool):
......@@ -43,6 +43,15 @@ def qa(skip_tests: bool, exitfirst: bool):
sys.exit(ret_code)
@dev.command(help='Runs tests and linting of the nomad gui source code. Useful before committing code.')
def gui_qa():
click.echo('Run gui code linting ...')
os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../gui')))
ret_code = 0
ret_code += os.system('yarn run eslint \'src/**/*.js\'')
sys.exit(ret_code)
@dev.command(help='Generates a JSON with all metainfo.')
def metainfo():
import json
......@@ -144,7 +153,6 @@ def legacy_metainfo(path):
'qbox.nomadmetainfo.json',
'quantum_espresso.nomadmetainfo.json',
'siesta.nomadmetainfo.json',
'skeleton.nomadmetainfo.json',
'turbomole.nomadmetainfo.json',
'vasp.nomadmetainfo.json',
'wien2k.nomadmetainfo.json',
......
......@@ -289,8 +289,8 @@ max_upload_size = 32 * (1024 ** 3)
raw_file_strip_cutoff = 1000
use_empty_parsers = False
reprocess_unmatched = True
aux_metadata_file = 'nomad_metadata'
aux_metadata_exts = ('json', 'yaml')
metadata_file_name = 'nomad'
metadata_file_extensions = ('json', 'yaml', 'yml')
def normalize_loglevel(value, default_level=logging.INFO):
......
......@@ -87,7 +87,7 @@ from .dft import DFTMetadata
from .ems import EMSMetadata
from .qcms import QCMSMetadata
from .datamodel import (
Dataset, User, EditableUserMetadata, UserProvidableMetadata, MongoMetadata,
Dataset, User, Author, EditableUserMetadata, UserProvidableMetadata, MongoMetadata,
EntryMetadata, EntryArchive)
from .optimade import OptimadeEntry, Species
from .metainfo import m_env
......@@ -107,7 +107,7 @@ domains = {
},
'ems': {
'metadata': EMSMetadata,
'metainfo_all_package': 'general_experimental',
'metainfo_all_package': 'common_experimental',
'root_section': 'section_experiment'
},
'qcms': {
......
......@@ -34,7 +34,7 @@ m_package = metainfo.Package()
from .encyclopedia import EncyclopediaMetadata # noqa
from .metainfo.public import section_run, Workflow # noqa
from .metainfo.general_experimental import section_experiment # noqa
from .metainfo.common_experimental import Experiment # noqa
from .metainfo.general_qcms import QuantumCMS # noqa
......@@ -56,8 +56,8 @@ class Author(metainfo.MSection):
derived=lambda user: ('%s %s' % (user.first_name, user.last_name)).strip(),
a_search=Search(mapping=Text(fields={'keyword': Keyword()})))
first_name = metainfo.Quantity(type=str)
last_name = metainfo.Quantity(type=str)
first_name = metainfo.Quantity(type=metainfo.Capitalized)
last_name = metainfo.Quantity(type=metainfo.Capitalized)
email = metainfo.Quantity(
type=str,
a_elastic=dict(mapping=Keyword), # TODO remove?
......@@ -547,7 +547,7 @@ class EntryMetadata(metainfo.MSection):
class EntryArchive(metainfo.MSection):
section_run = metainfo.SubSection(sub_section=section_run, repeats=True)
section_experiment = metainfo.SubSection(sub_section=section_experiment)
section_experiment = metainfo.SubSection(sub_section=Experiment)
section_quantum_cms = metainfo.SubSection(sub_section=QuantumCMS)
section_workflow = metainfo.SubSection(sub_section=Workflow)
section_metadata = metainfo.SubSection(sub_section=EntryMetadata)
......
......@@ -15,34 +15,41 @@
'''
Experimental material science specific metadata
'''
from nomad import config
from nomad.metainfo import Quantity, MSection, Section, Datetime
from nomad.metainfo.search_extension import Search
def _unavailable(value):
if value is None:
return config.services.unavailable_value
return value
class EMSMetadata(MSection):
m_def = Section(a_domain='ems')
# sample quantities
chemical = Quantity(type=str, default='not processed', a_search=Search())
sample_constituents = Quantity(type=str, default='not processed', a_search=Search())
sample_microstructure = Quantity(type=str, default='not processed', a_search=Search())
chemical = Quantity(type=str, a_search=Search())
sample_constituents = Quantity(type=str, a_search=Search())
sample_microstructure = Quantity(type=str, a_search=Search())
# general metadata
experiment_summary = Quantity(type=str, default='not processed', a_search=Search())
experiment_location = Quantity(type=str, default='not processed', a_search=Search())
experiment_time = Quantity(type=Datetime, a_search=Search())
experiment_summary = Quantity(type=str, a_search=Search())
origin_time = Quantity(type=Datetime, a_search=Search())
experiment_location = Quantity(type=str, a_search=Search())
# method
method = Quantity(type=str, default='not processed', a_search=Search())
probing_method = Quantity(type=str, default='not processed', a_search=Search())
method = Quantity(type=str, a_search=Search())
data_type = Quantity(type=str, a_search=Search())
probing_method = Quantity(type=str, a_search=Search())
# data metadata
repository_name = Quantity(type=str, default='not processed', a_search=Search())
repository_url = Quantity(type=str, default='not processed', a_search=Search())
entry_repository_url = Quantity(type=str, default='not processed', a_search=Search())
preview_url = Quantity(type=str, default='not processed', a_search=Search())
repository_name = Quantity(type=str, a_search=Search())
repository_url = Quantity(type=str, a_search=Search())
entry_repository_url = Quantity(type=str, a_search=Search())
preview_url = Quantity(type=str, a_search=Search())
# TODO move
quantities = Quantity(type=str, shape=['0..*'], default=[], a_search=Search())
......@@ -57,8 +64,12 @@ class EMSMetadata(MSection):
entry = self.m_parent
root_section = entry_archive.section_experiment
entry.formula = root_section.section_sample[0].sample_chemical_formula
atoms = root_section.section_sample[0].sample_atom_labels
entry.formula = root_section.section_sample.section_material.chemical_formula
atoms = root_section.section_sample.section_material.atom_labels
if atoms is None:
entry.atoms = []
else:
if hasattr(atoms, 'tolist'):
atoms = atoms.tolist()
entry.n_atoms = len(atoms)
......@@ -67,23 +78,34 @@ class EMSMetadata(MSection):
atoms.sort()
entry.atoms = atoms
self.chemical = root_section.section_sample[0].sample_chemical_name
self.sample_microstructure = root_section.section_sample[0].sample_microstructure
self.sample_constituents = root_section.section_sample[0].sample_constituents
self.chemical = _unavailable(root_section.section_sample.section_material.chemical_name)
self.sample_microstructure = _unavailable(root_section.section_sample.sample_microstructure)
self.sample_constituents = _unavailable(root_section.section_sample.sample_constituents)
self.experiment_summary = root_section.experiment_summary
self.experiment_location = root_section.experiment_location
experiment_time = root_section.experiment_time
if experiment_time != config.services.unavailable_value:
self.experiment_time = experiment_time
self.method = root_section.section_method[0].experiment_method_name
self.probing_method = root_section.section_method[0].probing_method
self.repository_name = root_section.section_data[0].data_repository_name
self.repository_url = root_section.section_data[0].data_repository_url
self.preview_url = root_section.section_data[0].data_preview_url
self.entry_repository_url = root_section.section_data[0].entry_repository_url
location = root_section.experiment_location
if location is not None:
location_str = ', '.join([
getattr(location, prop)
for prop in ['facility', 'institution', 'address']
if getattr(location, prop) is not None])
self.experiment_location = location_str
if root_section.experiment_time:
self.origin_time = root_section.experiment_time
elif root_section.experiment_publish_time:
self.origin_time = root_section.experiment_publish_time
else:
self.origin_time = self.m_parent.upload_time
self.data_type = _unavailable(root_section.section_method.data_type)
self.method = _unavailable(root_section.section_method.method_name)
self.probing_method = _unavailable(root_section.section_method.probing_method)
self.repository_name = _unavailable(root_section.section_data.repository_name)
self.repository_url = root_section.section_data.repository_url
self.preview_url = root_section.section_data.preview_url
self.entry_repository_url = root_section.section_data.entry_repository_url
self.group_hash = utils.hash(
entry.formula,
......@@ -99,3 +121,6 @@ class EMSMetadata(MSection):
quantities.add(property_def.name)
self.quantities = list(quantities)
if self.m_parent.references is None:
self.m_parent.references = [self.entry_repository_url]
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import numpy as np
from nomad.metainfo import (
MSection, Package, Quantity, SubSection, SectionProxy, Section,
Datetime, JSON)
m_package = Package(name='common_experimental')
class Experiment(MSection):
'''
The root section for all (meta)data that belongs to a single experiment.
'''
m_def = Section(validate=False)
experiment_summary = Quantity(
type=str, description='A descriptive summary of the content of the experiment.')
experiment_location = SubSection(sub_section=SectionProxy('Location'))
experiment_publish_time = Quantity(
type=Datetime, description='The datetime when this experiment was published.')
experiment_time = Quantity(
type=Datetime, description='The datetime of the beginning of the experiment.')
experiment_end_time = Quantity(
type=Datetime, description='The datetime of the experiment end.')
raw_metadata = Quantity(
type=JSON, description='The whole or partial metadata in its original source JSON format.')
section_data = SubSection(sub_section=SectionProxy('Data'))
section_method = SubSection(sub_section=SectionProxy('Method'))
section_sample = SubSection(sub_section=SectionProxy('Sample'))
class Location(MSection):
m_def = Section(validate=False)
address = Quantity(
type=str, description='''
The address where the experiment took place, format 'Country, City, Street'
''')
institution = Quantity(
type=str, description='''
Name of the institution hosting the experimental facility (e.g. in full or an
acronym).
''')
facility = Quantity(
type=str, description='''
Name of the experimental facility (e.g. in full or an acronym).
''')
class Data(MSection):
'''
This section contains information about the stored data.
'''
m_def = Section(validate=False)
repository_name = Quantity(
type=str, description='The name of the repository, where the data is stored.')
repository_url = Quantity(
type=str, description='An URL to the repository, where the data is stored.')
preview_url = Quantity(
type=str, description='An URL to an image file that contains a preview.')
entry_repository_url =