Skip to content
GitLab
Commit 32f9050b authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Refactored some aspects of ems domain. #403

parent 8f325955
Pipeline #81593 passed with stages
in 47 minutes and 14 seconds
Hide whitespace changes
Inline Side-by-side
.gitmodules
View file @ 32f9050b
......@@ -144,9 +144,6 @@
[submodule "dependencies/optimade-python-tools"]
path = dependencies/optimade-python-tools
url = https://github.com/markus1978/optimade-python-tools.git
[submodule "dependencies/parsers/eels"]
path = dependencies/parsers/eels
url = https://github.com/markus1978/eels.git
[submodule "dependencies/parsers/namd"]
path = dependencies/parsers/namd
url = https://gitlab.mpcdf.mpg.de/nomad-lab/parser-namd.git
......
aptfim @ b9b8f841
Compare c6bf6c2c...b9b8f841
Subproject commit c6bf6c2ca01bff208e424b531b8c214ab709b9b8
Subproject commit b9b8f841709736cbfcb3676226a89c06498aa26f
eels @ b42d4aae
Compare b42d4aae...b42d4aae
Subproject commit b42d4aae3fe431ed4afbbc738f5e19cec12634bd
eelsdb @ c4c75068
Compare 0d67544e...c4c75068
Subproject commit 0d67544ea06713b0cf333eeee8d6ef6929a47f5e
Subproject commit c4c75068f91f83fb9262eb09576710d11984db44
mpes @ aa7fbc09
Compare 0533ef83...aa7fbc09
Subproject commit 0533ef8393da2501226d60f4db9c0d7be32dbf81
Subproject commit aa7fbc09aa3bde6b654b764d21cd62eaf2badc9e
docs/dev/parser_tutorial.md
View file @ 32f9050b
......@@ -360,8 +360,8 @@ import json
from .metainfo import m_env
from nomad.parsing.parser import MatchingParser
from nomad.datamodel.metainfo.general_experimental import section_experiment as msection_experiment
from nomad.datamodel.metainfo.general_experimental import section_data as msection_data
from nomad.datamodel.metainfo.common_experimental import section_experiment as msection_experiment
from nomad.datamodel.metainfo.common_experimental import section_data as msection_data
from nomad.datamodel.metainfo.general_experimental_method import section_method as msection_method
from nomad.datamodel.metainfo.general_experimental_sample import section_sample as msection_sample
......
gui/src/components/domains.js
View file @ 32f9050b
......@@ -9,6 +9,7 @@ import {
import EMSVisualizations from './ems/EMSVisualizations'
import QCMSEntryOverview from './qcms/QCMSEntryOverview'
import QCMSEntryCards from './qcms/QCMSEntryCards'
import { Link } from '@material-ui/core'
/* eslint-disable react/display-name */
......@@ -201,20 +202,32 @@ export const domains = ({
*/
searchResultColumns: {
'formula': {
label: 'Formula'
label: 'Formula',
supportsSort: true
},
'ems.chemical': {
label: 'Chemical',
supportsSort: true
},
'ems.method': {
label: 'Method'
label: 'Method',
supportsSort: true
},
'ems.data_type': {
label: 'Data',
supportsSort: true
},
'ems.experiment_location': {
label: 'Location'
'ems.origin_time': {
label: 'Date',
supportsSort: true,
render: entry => (entry.ems && entry.ems.origin_time && new Date(entry.ems.origin_time).toLocaleDateString()) || 'unavailable'
},
'ems.experiment_time': {
label: 'Date/Time',
render: entry => (entry.ems && entry.ems.experiment_time !== 'unavailable') ? new Date(entry.ems.experiment_time * 1000).toLocaleString() : 'unavailable'
'ems.repository_url': {
label: 'Source',
render: entry => <Link target="external" href={entry.ems.entry_repository_url}>{entry.ems.repository_url}</Link>
}
},
defaultSearchResultColumns: ['formula', 'ems.method', 'ems.experiment_location', 'ems.experiment_time'],
defaultSearchResultColumns: ['formula', 'ems.chemical', 'ems.method', 'ems.data_type', 'ems.origin_time', 'ems.repository_url'],
/**
* A component to render the domain specific quantities in the metadata card of
* the entry view. Needs to work with props: data (the entry data from the API),
......
gui/src/components/ems/EMSEntryCards.js
View file @ 32f9050b
......@@ -27,7 +27,7 @@ class EMSEntryCards extends React.Component {
<CardContent classes={{root: classes.cardContent}}>
<Markdown classes={{root: classes.description}}>{`
The data for this experiment is externally stored and managed. Download the raw experiment data:
[${data.ems.repository_url}](${data.ems.repository_url}).
[${data.ems && data.ems.repository_url}](${data.ems && data.ems.entry_repository_url}).
The meta data describing this experiment in its original format, can be
downloaded here directly:
......
gui/src/components/ems/EMSEntryOverview.js
View file @ 32f9050b
import React from 'react'
import PropTypes from 'prop-types'
import Quantity from '../Quantity'
import { Typography } from '@material-ui/core'
import { Typography, Link } from '@material-ui/core'
import { apiBase } from '../../config'
export default class EMSEntryOverview extends React.Component {
......@@ -46,6 +46,15 @@ export default class EMSEntryOverview extends React.Component {
return (
<Quantity column>
<Quantity quantity="ems.experiment_summary" label="summary" {...this.props} />
{this.state.previewBroken
? data.ems.entry_repository_url && <Quantity label="preview" {...this.props}>
<Typography noWrap>
<a target="external" href={ems.entry_repository_url}>visit this entry on the external database</a>
</Typography>
</Quantity>
: <Quantity label="preview" {...this.props}>
<img alt="preview" style={{maxWidth: '100%', height: 'auto'}} src={relative_preview_url} onError={this.handleBrokenPreview}></img>
</Quantity>}
<Quantity row>
<Quantity column>
<Quantity row>
......@@ -56,26 +65,17 @@ export default class EMSEntryOverview extends React.Component {
</Quantity>
<Quantity quantity="ems.method" label="experimental method" noWrap {...this.props} />
<Quantity quantity="ems.experiment_location" label="experiment location" noWrap {...this.props} />
<Quantity label="experiment time" {...this.props}>
<Quantity label="experiment or experiment publish date" {...this.props}>
<Typography noWrap>{
data.ems.experiment_time && data.ems.experiment_time !== 'unavailable' ? new Date(data.ems.experiment_time * 1000).toLocaleString() : 'unavailable'
(ems && ems.origin_time && new Date(ems.origin_time).toLocaleDateString()) || 'unavailable'
}</Typography>
</Quantity>
<Quantity label="data" {...this.props}>
<Typography noWrap>
<a target="external" href={data.ems.repository_url}>{data.ems.repository_name}</a>
<Link target="external" href={ems.entry_repository_url}>{ems.repository_url}</Link>
</Typography>
</Quantity>
</Quantity>
{this.state.previewBroken
? data.ems.entry_repository_url && <Quantity label="preview" {...this.props}>
<Typography noWrap>
<a target="external" href={data.ems.entry_repository_url}>visit this entry on the external database</a>
</Typography>
</Quantity>
: <Quantity label="preview" {...this.props}>
<img alt="preview" style={{maxWidth: '100%', height: 'auto'}} src={relative_preview_url} onError={this.handleBrokenPreview}></img>
</Quantity>}
</Quantity>
</Quantity>
)
......
gui/src/components/entry/RepoEntryView.js
View file @ 32f9050b
import React from 'react'
import React, { useContext, useState, useEffect } from 'react'
import PropTypes from 'prop-types'
import { withStyles, Divider, Card, CardContent, Grid, CardHeader, Typography, Link } from '@material-ui/core'
import { withApi } from '../api'
import { compose } from 'recompose'
import { Divider, Card, CardContent, Grid, CardHeader, Typography, Link, makeStyles } from '@material-ui/core'
import { apiContext } from '../api'
import ApiDialogButton from '../ApiDialogButton'
// import Structure from '../visualization/Structure'
import Quantity from '../Quantity'
......@@ -10,170 +9,152 @@ import { Link as RouterLink } from 'react-router-dom'
import { DOI } from '../search/DatasetList'
import { domains } from '../domains'
import { EntryPageContent } from './EntryPage'
import { errorContext } from '../errors'
class RepoEntryView extends React.Component {
static styles = theme => ({
root: {
marginTop: theme.spacing(2)
},
error: {
marginTop: theme.spacing(2)
},
cardContent: {
paddingTop: 0
},
entryCards: {
marginTop: theme.spacing(2)
},
structureViewer: {
height: '25rem',
padding: '0'
}
})
static propTypes = {
classes: PropTypes.object.isRequired,
api: PropTypes.object.isRequired,
raiseError: PropTypes.func.isRequired,
uploadId: PropTypes.string.isRequired,
calcId: PropTypes.string.isRequired
}
static defaultState = {
calcData: null,
doesNotExist: false
const useStyles = makeStyles(theme => ({
root: {
marginTop: theme.spacing(2)
},
error: {
marginTop: theme.spacing(2)
},
cardContent: {
paddingTop: 0
},
entryCards: {
marginTop: theme.spacing(2)
},
structureViewer: {
height: '25rem',
padding: '0'
}
}))
state = {...RepoEntryView.defaultState}
componentDidMount() {
this.update()
}
export default function RepoEntryView({uploadId, calcId}) {
const classes = useStyles()
const {api} = useContext(apiContext)
const {raiseError} = useContext(errorContext)
const [state, setState] = useState({calcData: null, doesNotExist: false})
componentDidUpdate(prevProps) {
if (prevProps.api !== this.props.api ||
prevProps.uploadId !== this.props.uploadId ||
prevProps.calcId !== this.props.calcId) {
this.setState({...RepoEntryView.defaultState})
this.update()
}
}
useEffect(() => {
setState({calcData: null, doesNotExist: false})
}, [setState, uploadId, calcId])
update() {
const {uploadId, calcId} = this.props
this.props.api.repo(uploadId, calcId).then(data => {
this.setState({calcData: data, doesNotExist: false})
useEffect(() => {
api.repo(uploadId, calcId).then(data => {
setState({calcData: data, doesNotExist: false})
}).catch(error => {
this.setState({calcData: null, doesNotExist: false})
if (error.name === 'DoesNotExist') {
this.setState({doesNotExist: true})
setState({calcData: null, doesNotExist: true})
} else {
this.props.raiseError(error)
setState({calcData: null, doesNotExist: false})
raiseError(error)
}
})
}
}, [api, raiseError, uploadId, calcId, setState])
render() {
const { classes, ...calcProps } = this.props
const calcData = this.state.calcData || calcProps
const loading = !this.state.calcData
const { uploadId, calcId } = calcProps
const quantityProps = {data: calcData, loading: loading}
const calcData = state.calcData || {uploadId: uploadId, calcId: calcId}
const loading = !state.calcData
const quantityProps = {data: calcData, loading: loading}
const authors = loading ? null : calcData.authors
const domain = calcData.domain && domains[calcData.domain]
const authors = loading ? null : calcData.authors
const domain = calcData.domain && domains[calcData.domain]
let entryHeader = 'Entry metadata'
if (domain) {
entryHeader = domain.entryTitle(calcData)
}
let entryHeader = 'Entry metadata'
if (domain) {
entryHeader = domain.entryTitle(calcData)
}
if (this.state.doesNotExist) {
return <Typography className={classes.error}>
This entry does not exist.
if (state.doesNotExist) {
return <EntryPageContent className={classes.root} fixed>
<Typography className={classes.error}>
This entry does not exist.
</Typography>
}
</EntryPageContent>
}
return (
<EntryPageContent className={classes.root} fixed>
<Grid container spacing={2}>
<Grid item xs={7}>
<Card>
<CardHeader
title={entryHeader}
action={<ApiDialogButton title="Repository JSON" data={calcData} />}
/>
<CardContent classes={{root: classes.cardContent}}>
{domain && <domain.EntryOverview data={calcData} loading={loading} />}
</CardContent>
<Divider />
<CardContent>
<Quantity column>
<Quantity quantity='comment' placeholder='no comment' {...quantityProps} />
<Quantity quantity='references' placeholder='no references' {...quantityProps}>
{calcData.references &&
<div style={{display: 'inline-grid'}}>
{calcData.references.map(ref => <Typography key={ref} noWrap>
<a href={ref}>{ref}</a>
</Typography>)}
</div>}
</Quantity>
<Quantity quantity='authors' {...quantityProps}>
<Typography>
{(authors || []).map(author => author.name).join('; ')}
</Typography>
</Quantity>
<Quantity quantity='datasets' placeholder='no datasets' {...quantityProps}>
{calcData.datasets &&
<div>
{calcData.datasets.map(ds => (
<Typography key={ds.dataset_id}>
<Link component={RouterLink} to={`/dataset/id/${ds.dataset_id}`}>{ds.name}</Link>
{ds.doi ? <span>&nbsp; (<DOI doi={ds.doi}/>)</span> : ''}
</Typography>))}
</div>}
</Quantity>
return (
<EntryPageContent className={classes.root} fixed>
<Grid container spacing={2}>
<Grid item xs={7}>
<Card>
<CardHeader
title={entryHeader}
action={<ApiDialogButton title="Repository JSON" data={calcData} />}
/>
<CardContent classes={{root: classes.cardContent}}>
{domain && <domain.EntryOverview data={calcData} loading={loading} />}
</CardContent>
<Divider />
<CardContent>
<Quantity column>
<Quantity quantity='comment' placeholder='no comment' {...quantityProps} />
<Quantity quantity='references' placeholder='no references' {...quantityProps}>
{calcData.references &&
<div style={{display: 'inline-grid'}}>
{calcData.references.map(ref => <Typography key={ref} noWrap>
<a href={ref}>{ref}</a>
</Typography>)}
</div>}
</Quantity>
<Quantity quantity='authors' {...quantityProps}>
<Typography>
{(authors || []).map(author => author.name).join('; ')}
</Typography>
</Quantity>
</CardContent>
</Card>
</Grid>
<Quantity quantity='datasets' placeholder='no datasets' {...quantityProps}>
{calcData.datasets &&
<div>
{calcData.datasets.map(ds => (
<Typography key={ds.dataset_id}>
<Link component={RouterLink} to={`/dataset/id/${ds.dataset_id}`}>{ds.name}</Link>
{ds.doi ? <span>&nbsp; (<DOI doi={ds.doi}/>)</span> : ''}
</Typography>))}
</div>}
</Quantity>
</Quantity>
</CardContent>
</Card>
</Grid>
<Grid item xs={5}>
<Card>
<CardHeader title="Ids / processing" />
<CardContent classes={{root: classes.cardContent}}>
<Quantity column style={{maxWidth: 350}}>
<Quantity quantity="calc_id" label={`${domain ? domain.entryLabel : 'entry'} id`} noWrap withClipboard {...quantityProps} />
<Quantity quantity={entry => entry.encyclopedia.material.material_id} label='material id' loading={loading} noWrap {...quantityProps} withClipboard />
<Quantity quantity="raw_id" label='raw id' loading={loading} noWrap {...quantityProps} withClipboard />
<Quantity quantity="external_id" label='external id' loading={loading} noWrap {...quantityProps} withClipboard />
<Quantity quantity="mainfile" loading={loading} noWrap ellipsisFront {...quantityProps} withClipboard />
<Quantity quantity="upload_id" label='upload id' {...quantityProps} noWrap withClipboard />
<Quantity quantity="upload_time" label='upload time' noWrap {...quantityProps} >
<Typography noWrap>
{new Date(calcData.upload_time).toLocaleString()}
</Typography>
</Quantity>
<Quantity quantity="last_processing" label='last processing' loading={loading} placeholder="not processed" noWrap {...quantityProps}>
<Typography noWrap>
{new Date(calcData.last_processing).toLocaleString()}
</Typography>
</Quantity>
<Quantity quantity="last_processing" label='processing version' loading={loading} noWrap placeholder="not processed" {...quantityProps}>
<Typography noWrap>
{calcData.nomad_version}/{calcData.nomad_commit}
</Typography>
</Quantity>
<Grid item xs={5}>
<Card>
<CardHeader title="Ids / processing" />
<CardContent classes={{root: classes.cardContent}}>
<Quantity column style={{maxWidth: 350}}>
<Quantity quantity="calc_id" label={`${domain ? domain.entryLabel : 'entry'} id`} noWrap withClipboard {...quantityProps} />
<Quantity quantity={entry => entry.encyclopedia.material.material_id} label='material id' loading={loading} noWrap {...quantityProps} withClipboard />
<Quantity quantity="raw_id" label='raw id' loading={loading} noWrap {...quantityProps} withClipboard />
<Quantity quantity="external_id" label='external id' loading={loading} noWrap {...quantityProps} withClipboard />
<Quantity quantity="mainfile" loading={loading} noWrap ellipsisFront {...quantityProps} withClipboard />
<Quantity quantity="upload_id" label='upload id' {...quantityProps} noWrap withClipboard />
<Quantity quantity="upload_time" label='upload time' noWrap {...quantityProps} >
<Typography noWrap>
{new Date(calcData.upload_time).toLocaleString()}
</Typography>
</Quantity>
</CardContent>
</Card>
</Grid>
<Quantity quantity="last_processing" label='last processing' loading={loading} placeholder="not processed" noWrap {...quantityProps}>
<Typography noWrap>
{new Date(calcData.last_processing).toLocaleString()}
</Typography>
</Quantity>
<Quantity quantity="last_processing" label='processing version' loading={loading} noWrap placeholder="not processed" {...quantityProps}>
<Typography noWrap>
{calcData.nomad_version}/{calcData.nomad_commit}
</Typography>
</Quantity>
</Quantity>
</CardContent>
</Card>
</Grid>
</Grid>
{domain && <domain.EntryCards data={calcData} calcId={calcId} uploadId={uploadId} classes={{root: classes.entryCards}} />}
</EntryPageContent>
)
}
{domain && <domain.EntryCards data={calcData} calcId={calcId} uploadId={uploadId} classes={{root: classes.entryCards}} />}
</EntryPageContent>
)
}
export default compose(withApi(false, true), withStyles(RepoEntryView.styles))(RepoEntryView)
RepoEntryView.propTypes = {
uploadId: PropTypes.string.isRequired,
calcId: PropTypes.string.isRequired
}
nomad/cli/dev.py
View file @ 32f9050b
......@@ -24,7 +24,7 @@ def dev():
pass
@dev.command(help='Runs tests and linting of the nomad source code. Useful before committing code.')
@dev.command(help='Runs tests and linting of the nomad python source code. Useful before committing code.')
@click.option('--skip-tests', help='Do no tests, just do code checks.', is_flag=True)
@click.option('-x', '--exitfirst', help='Stop testing after first failed test case.', is_flag=True)
def qa(skip_tests: bool, exitfirst: bool):
......@@ -43,6 +43,15 @@ def qa(skip_tests: bool, exitfirst: bool):
sys.exit(ret_code)
@dev.command(help='Runs tests and linting of the nomad gui source code. Useful before committing code.')
def gui_qa():
click.echo('Run gui code linting ...')
os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../gui')))
ret_code = 0
ret_code += os.system('yarn run eslint \'src/**/*.js\'')
sys.exit(ret_code)
@dev.command(help='Generates a JSON with all metainfo.')
def metainfo():
import json
......@@ -144,7 +153,6 @@ def legacy_metainfo(path):
'qbox.nomadmetainfo.json',
'quantum_espresso.nomadmetainfo.json',
'siesta.nomadmetainfo.json',
'skeleton.nomadmetainfo.json',
'turbomole.nomadmetainfo.json',
'vasp.nomadmetainfo.json',
'wien2k.nomadmetainfo.json',
......
nomad/config.py
View file @ 32f9050b
......@@ -289,8 +289,8 @@ max_upload_size = 32 * (1024 ** 3)
raw_file_strip_cutoff = 1000
use_empty_parsers = False
reprocess_unmatched = True
aux_metadata_file = 'nomad_metadata'
aux_metadata_exts = ('json', 'yaml')
metadata_file_name = 'nomad'
metadata_file_extensions = ('json', 'yaml', 'yml')
def normalize_loglevel(value, default_level=logging.INFO):
......
nomad/datamodel/__init__.py
View file @ 32f9050b
......@@ -87,7 +87,7 @@ from .dft import DFTMetadata
from .ems import EMSMetadata
from .qcms import QCMSMetadata
from .datamodel import (
Dataset, User, EditableUserMetadata, UserProvidableMetadata, MongoMetadata,
Dataset, User, Author, EditableUserMetadata, UserProvidableMetadata, MongoMetadata,
EntryMetadata, EntryArchive)
from .optimade import OptimadeEntry, Species
from .metainfo import m_env
......@@ -107,7 +107,7 @@ domains = {
},
'ems': {
'metadata': EMSMetadata,
'metainfo_all_package': 'general_experimental',
'metainfo_all_package': 'common_experimental',
'root_section': 'section_experiment'
},
'qcms': {
......
nomad/datamodel/datamodel.py
View file @ 32f9050b
......@@ -34,7 +34,7 @@ m_package = metainfo.Package()
from .encyclopedia import EncyclopediaMetadata # noqa
from .metainfo.public import section_run, Workflow # noqa
from .metainfo.general_experimental import section_experiment # noqa
from .metainfo.common_experimental import Experiment # noqa
from .metainfo.general_qcms import QuantumCMS # noqa
......@@ -56,8 +56,8 @@ class Author(metainfo.MSection):
derived=lambda user: ('%s %s' % (user.first_name, user.last_name)).strip(),
a_search=Search(mapping=Text(fields={'keyword': Keyword()})))
first_name = metainfo.Quantity(type=str)
last_name = metainfo.Quantity(type=str)
first_name = metainfo.Quantity(type=metainfo.Capitalized)
last_name = metainfo.Quantity(type=metainfo.Capitalized)
email = metainfo.Quantity(
type=str,
a_elastic=dict(mapping=Keyword), # TODO remove?
......@@ -547,7 +547,7 @@ class EntryMetadata(metainfo.MSection):
class EntryArchive(metainfo.MSection):
section_run = metainfo.SubSection(sub_section=section_run, repeats=True)
section_experiment = metainfo.SubSection(sub_section=section_experiment)
section_experiment = metainfo.SubSection(sub_section=Experiment)
section_quantum_cms = metainfo.SubSection(sub_section=QuantumCMS)
section_workflow = metainfo.SubSection(sub_section=Workflow)
section_metadata = metainfo.SubSection(sub_section=EntryMetadata)
......
nomad/datamodel/ems.py
View file @ 32f9050b
......@@ -15,34 +15,41 @@
'''
Experimental material science specific metadata
'''
from nomad import config
from nomad.metainfo import Quantity, MSection, Section, Datetime
from nomad.metainfo.search_extension import Search
def _unavailable(value):
if value is None:
return config.services.unavailable_value
return value