Commit 632d7fd2 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added quantum computational domain and archive parser.

parent cd8c3b8b
Pipeline #81515 failed with stages
in 33 minutes and 36 seconds
......@@ -96,12 +96,17 @@ export const rootSections = sortDefs(defs.filter(def => (
))
export function resolveRef(ref, data) {
data = data || metainfo
const segments = ref.split('/').filter(segment => segment !== '')
const reducer = (current, segment) => {
return isNaN(segment) ? current[segment] : current[parseInt(segment)]
try {
data = data || metainfo
const segments = ref.split('/').filter(segment => segment !== '')
const reducer = (current, segment) => {
return isNaN(segment) ? current[segment] : current[parseInt(segment)]
}
return segments.reduce(reducer, data)
} catch (e) {
console.log('could not resolve: ' + ref)
throw e
}
return segments.reduce(reducer, data)
}
export function metainfoDef(name) {
......
......@@ -7,12 +7,14 @@ import {
DFTSystemVisualizations, DFTPropertyVisualizations, DFTMethodVisualizations
} from './dft/DFTVisualizations'
import EMSVisualizations from './ems/EMSVisualizations'
import QCMSEntryOverview from './qcms/QCMSEntryOverview'
import QCMSEntryCards from './qcms/QCMSEntryCards'
/* eslint-disable react/display-name */
export const domains = ({
dft: {
name: 'computational',
name: 'computational data',
label: 'Computational material science data',
key: 'dft',
about: 'This include data from many computational material science codes',
......@@ -160,9 +162,9 @@ export const domains = ({
searchTabs: ['entries', 'materials', 'datasets', 'groups', 'uploads']
},
ems: {
name: 'experimental',
name: 'experimental data',
key: 'ems',
label: 'Material science experiment data',
label: 'Experimental material science data',
about: 'This is metadata from material science experiments',
entryLabel: 'entry',
entryLabelPlural: 'entries',
......@@ -229,5 +231,64 @@ export const domains = ({
* Names of the possible search tabs for this domain
*/
searchTabs: ['entries', 'datasets', 'uploads']
},
qcms: {
name: 'quantum computational data',
key: 'qcms',
label: 'Quantum computational material science data',
about: 'This is computational material science data calculated by quantum computers',
entryLabel: 'calculation',
entryLabelPlural: 'calculations',
entryTitle: () => 'Quantum computer calculation',
searchPlaceholder: 'enter atoms',
searchVisualizations: {
},
/**
* Metrics are used to show values for aggregations. Each metric has a key (used
* for API calls), a label (used in the select form), and result string (to show
* the overall amount in search results).
*/
searchMetrics: {
code_runs: {
label: 'Calculations',
tooltip: 'Statistics will show the number of entires; usually each entry represents a single calculation.',
renderResultString: count => (<span><b>{count}</b> entries</span>)
},
datasets: {
label: 'Datasets',
tooltip: 'Shows statistics in terms of datasets that entries belong to.',
renderResultString: count => (<span> curated in <b>{count}</b> datasets</span>)
}
},
defaultSearchMetric: 'code_runs',
/**
* An dict where each object represents a column. Possible keys are label, render.
* Default render
*/
searchResultColumns: {
'formula': {
label: 'Formula'
},
'qcms.chemical': {
label: 'Chemical name'
}
},
defaultSearchResultColumns: ['formula', 'qcms.chemical'],
/**
* A component to render the domain specific quantities in the metadata card of
* the entry view. Needs to work with props: data (the entry data from the API),
* loading (a bool with api loading status).
*/
EntryOverview: QCMSEntryOverview,
/**
* A component to render additional domain specific cards in the
* the entry view. Needs to work with props: data (the entry data from the API),
* loading (a bool with api loading status).
*/
EntryCards: QCMSEntryCards,
/**
* Names of the possible search tabs for this domain
*/
searchTabs: ['entries', 'datasets', 'uploads']
}
})
import React from 'react'
import { Card, CardHeader, CardContent, makeStyles } from '@material-ui/core'
import RawFiles from '../entry/RawFiles'
const useStyles = makeStyles(theme => ({
root: {
marginTop: theme.spacing(2)
}
}))
export default function QCMSEntryCards(props) {
const classes = useStyles()
return (
<Card className={classes.root}>
<CardHeader title="Raw files" />
<CardContent>
<RawFiles {...props} />
</CardContent>
</Card>
)
}
import React from 'react'
import PropTypes from 'prop-types'
import Quantity from '../Quantity'
import { Typography } from '@material-ui/core'
export default function QCMSEntryOverview(props) {
if (!props.data) {
return <Typography color="error">No metadata available</Typography>
}
return (
<Quantity column>
<Quantity row>
<Quantity quantity="formula" label="formula" noWrap {...props} />
<Quantity quantity="qcms.chemical" label="chemical name" noWrap {...props} />
</Quantity>
</Quantity>
)
}
QCMSEntryOverview.propTypes = {
data: PropTypes.object.isRequired,
loading: PropTypes.bool
}
......@@ -411,7 +411,7 @@ DomainSelect.propTypes = {
const ownerLabel = {
all: 'All entries',
visible: 'Include your private entries',
visible: 'with private entries',
public: 'Only public entries',
user: 'Only your entries',
shared: 'Incl. shared data',
......
......@@ -43,12 +43,14 @@ def parse(
entry_archive = datamodel.EntryArchive()
metadata = entry_archive.m_create(datamodel.EntryMetadata)
metadata.domain = parser.domain
try:
parser.parse(mainfile_path, entry_archive, logger=logger)
except Exception as e:
logger.error('parsing was not successful', exc_info=e)
if metadata.domain is None:
metadata.domain = parser.domain
logger.info('ran parser')
return entry_archive
......
......@@ -85,13 +85,17 @@ from nomad.metainfo import Environment
from .dft import DFTMetadata
from .ems import EMSMetadata
from .datamodel import Dataset, User, EditableUserMetadata, MongoMetadata, EntryMetadata, EntryArchive
from .qcms import QCMSMetadata
from .datamodel import (
Dataset, User, EditableUserMetadata, UserProvidableMetadata, MongoMetadata,
EntryMetadata, EntryArchive)
from .optimade import OptimadeEntry, Species
from .metainfo import m_env
m_env.m_add_sub_section(Environment.packages, sys.modules['nomad.datamodel.datamodel'].m_package) # type: ignore
m_env.m_add_sub_section(Environment.packages, sys.modules['nomad.datamodel.dft'].m_package) # type: ignore
m_env.m_add_sub_section(Environment.packages, sys.modules['nomad.datamodel.ems'].m_package) # type: ignore
m_env.m_add_sub_section(Environment.packages, sys.modules['nomad.datamodel.qcms'].m_package) # type: ignore
m_env.m_add_sub_section(Environment.packages, sys.modules['nomad.datamodel.encyclopedia'].m_package) # type: ignore
m_env.m_add_sub_section(Environment.packages, sys.modules['nomad.datamodel.optimade'].m_package) # type: ignore
......@@ -103,8 +107,13 @@ domains = {
},
'ems': {
'metadata': EMSMetadata,
'metainfo_all_package': 'all.experimental.nomadmetainfo.json',
'metainfo_all_package': 'general_experimental',
'root_section': 'section_experiment'
},
'qcms': {
'metadata': QCMSMetadata,
'metainfo_all_package': 'general_qcms',
'root_section': 'section_quantum_cms'
}
}
......
......@@ -26,6 +26,7 @@ from nomad.metainfo.mongoengine_extension import Mongo, MongoDocument
from .dft import DFTMetadata
from .ems import EMSMetadata
from .qcms import QCMSMetadata
# This is usually defined automatically when the first metainfo definition is evaluated, but
# due to the next imports requireing the m_package already, this would be too late.
......@@ -34,6 +35,7 @@ m_package = metainfo.Package()
from .encyclopedia import EncyclopediaMetadata # noqa
from .metainfo.public import section_run, Workflow # noqa
from .metainfo.general_experimental import section_experiment # noqa
from .metainfo.general_qcms import QuantumCMS # noqa
def _only_atoms(atoms):
......@@ -223,8 +225,13 @@ class DatasetReference(metainfo.Reference):
dataset_reference = DatasetReference()
class UserProvidableMetadata(metainfo.MCategory):
''' NOMAD entry metadata quantities that can be determined by the user, e.g. via nomad.yaml. '''
class EditableUserMetadata(metainfo.MCategory):
''' NOMAD entry quantities that can be edited by the user after publish. '''
''' NOMAD entry metadata quantities that can be edited by the user after publish. '''
m_def = metainfo.Category(categories=[UserProvidableMetadata])
class MongoMetadata(metainfo.MCategory):
......@@ -342,13 +349,13 @@ class EntryMetadata(metainfo.MSection):
raw_id = metainfo.Quantity(
type=str,
description='A raw format specific id that was acquired from the files of this entry',
categories=[MongoMetadata],
categories=[MongoMetadata, UserProvidableMetadata],
a_search=Search(many_or='append'))
domain = metainfo.Quantity(
type=metainfo.MEnum('dft', 'ems'),
type=metainfo.MEnum('dft', 'ems', 'qcms'),
description='The material science domain',
categories=[MongoMetadata],
categories=[MongoMetadata, UserProvidableMetadata],
a_search=Search())
published = metainfo.Quantity(
......@@ -395,7 +402,7 @@ class EntryMetadata(metainfo.MSection):
a_search=Search())
external_db = metainfo.Quantity(
type=metainfo.MEnum('EELSDB'), categories=[MongoMetadata],
type=metainfo.MEnum('EELSDB'), categories=[MongoMetadata, UserProvidableMetadata],
description='The repository or external database where the original entry resides.',
a_search=Search())
......@@ -482,7 +489,7 @@ class EntryMetadata(metainfo.MSection):
description='Search for a particular dataset by its id.')])
external_id = metainfo.Quantity(
type=str, categories=[MongoMetadata],
type=str, categories=[MongoMetadata, UserProvidableMetadata],
description='A user provided external id.',
a_search=Search(many_or='split'))
......@@ -515,6 +522,7 @@ class EntryMetadata(metainfo.MSection):
ems = metainfo.SubSection(sub_section=EMSMetadata, a_search='ems')
dft = metainfo.SubSection(sub_section=DFTMetadata, a_search='dft')
qcms = metainfo.SubSection(sub_section=QCMSMetadata, a_search='qcms')
encyclopedia = metainfo.SubSection(sub_section=EncyclopediaMetadata, a_search='encyclopedia')
def apply_user_metadata(self, metadata: dict):
......@@ -522,8 +530,7 @@ class EntryMetadata(metainfo.MSection):
self.m_update(**metadata)
def apply_domain_metadata(self, archive):
"""Used to apply metadata that is related to the domain.
"""
''' Used to apply metadata that is related to the domain. '''
assert self.domain is not None, 'all entries must have a domain'
domain_sub_section_def = self.m_def.all_sub_sections.get(self.domain)
domain_section_def = domain_sub_section_def.sub_section
......@@ -541,6 +548,7 @@ class EntryArchive(metainfo.MSection):
section_run = metainfo.SubSection(sub_section=section_run, repeats=True)
section_experiment = metainfo.SubSection(sub_section=section_experiment)
section_quantum_cms = metainfo.SubSection(sub_section=QuantumCMS)
section_workflow = metainfo.SubSection(sub_section=Workflow)
section_metadata = metainfo.SubSection(sub_section=EntryMetadata)
......
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from nomad.metainfo import Environment
from nomad.metainfo.legacy import LegacyMetainfoEnvironment
......
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import numpy as np # pylint: disable=unused-import
import typing # pylint: disable=unused-import
from nomad.metainfo import ( # pylint: disable=unused-import
......
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import numpy as np # pylint: disable=unused-import
import typing # pylint: disable=unused-import
from nomad.metainfo import ( # pylint: disable=unused-import
......
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from nomad.metainfo import MSection, Quantity, Package
m_package = Package()
class QuantumCMS(MSection):
'''
The root section for all (meta)data that belongs to a single calculation.
'''
chemical_formula = Quantity(
type=str,
description=''' The chemical formula that describes the simulated material ''')
chemical_name = Quantity(
type=str,
description=''' The chemical name that describes the simulated material ''')
atom_labels = Quantity(
type=str, shape=['1..*'],
description=''' Labels for the atoms/elements that comprise the simulated material ''')
m_package.__init_metainfo__()
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
'''
Quantum computational materials science metadata
'''
from nomad import config
from nomad.metainfo import Quantity, MSection, Section
from nomad.metainfo.search_extension import Search
class QCMSMetadata(MSection):
m_def = Section(a_domain='qcms')
# sample quantities
chemical = Quantity(type=str, default='not processed', a_search=Search())
# TODO move
quantities = Quantity(type=str, shape=['0..*'], default=[], a_search=Search())
def apply_domain_metadata(self, entry_archive):
if entry_archive is None:
return
entry = self.m_parent
root_section = entry_archive.section_quantum_cms
entry.formula = root_section.chemical_formula
if not entry.formula:
entry.formula = config.services.unavailable_value
atoms = root_section.atom_labels
if hasattr(atoms, 'tolist'):
atoms = atoms.tolist()
entry.n_atoms = len(atoms)
atoms = list(set(atoms))
atoms.sort()
entry.atoms = atoms
self.chemical = root_section.chemical_name
if not self.chemical:
self.chemical = config.services.unavailable_value
quantities = set()
quantities.add(root_section.m_def.name)
for _, property_def, _ in root_section.m_traverse():
quantities.add(property_def.name)
self.quantities = list(quantities)
......@@ -16,8 +16,9 @@ from typing import List
from abc import ABCMeta, abstractmethod
import re
from nomad import config
from nomad.metainfo import Environment
from nomad.datamodel import EntryArchive
from nomad.datamodel import EntryArchive, UserProvidableMetadata, EntryMetadata
class Parser(metaclass=ABCMeta):
......@@ -166,6 +167,35 @@ class FairdiParser(MatchingParser):
return archive
class ArchiveParser(FairdiParser):
def __init__(self):
super().__init__(
name='parsers/archive',
code_name=config.services.unavailable_value,
domain=None,
mainfile_mime_re=r'application/json',
mainfile_name_re=r'.*archive\.json',
mainfile_contents_re=r'section_')
def parse(self, mainfile: str, archive: EntryArchive, logger=None):
import json
with open(mainfile, 'rt') as f:
archive_data = json.load(f)
metadata_data = archive_data.get(EntryArchive.section_metadata.name, None)
if metadata_data is not None:
metadata = archive.section_metadata
for key, value in metadata_data.items():
if UserProvidableMetadata.m_def not in getattr(EntryMetadata, key).categories:
continue
metadata.m_update_from_dict({key: value})
del(archive_data[EntryArchive.section_metadata.name])
archive.m_update_from_dict(archive_data)
class MissingParser(MatchingParser):
'''
A parser implementation that just fails and is used to match mainfiles with known
......
......@@ -17,7 +17,7 @@ import os.path
from nomad import config, datamodel
from .parser import MissingParser, BrokenParser, Parser
from .parser import MissingParser, BrokenParser, Parser, ArchiveParser
from .legacy import LegacyParser, VaspOutcarParser
from .artificial import EmptyParser, GenerateRandomParser, TemplateParser, ChaosParser
......@@ -445,7 +445,8 @@ parsers = [
parser_class_name='mopacparser.MopacParser',
mainfile_contents_re=r'\s*\*\*\s*MOPAC\s*([0-9a-zA-Z]*)\s*\*\*\s*',
mainfile_mime_re=r'text/.*',
)
),
ArchiveParser()
]
empty_parsers = [
......
......@@ -174,7 +174,9 @@ class Calc(Proc):
'''
entry_metadata = datamodel.EntryMetadata()
if self.parser is not None:
entry_metadata.domain = parser_dict[self.parser].domain
parser = parser_dict[self.parser]
if parser.domain:
entry_metadata.domain = parser_dict[self.parser].domain
entry_metadata.upload_id = self.upload_id
entry_metadata.calc_id = self.calc_id
entry_metadata.mainfile = self.mainfile
......
{
"section_quantum_cms": {
"atom_labels": [ "He"],
"chemical_formula": "He2",
"chemical_name": "Helium"
},
"section_metadata": {
"domain": "qcms"
}
}
\ No newline at end of file
......@@ -453,6 +453,18 @@ def test_ems_data(proc_infra, test_user):
assert_search_upload(entries, additional_keys, published=False)
def test_qcms_data(proc_infra, test_user):
upload = run_processing(('test_qcms_upload', 'tests/data/proc/examples_qcms.zip'), test_user)
additional_keys = ['qcms.chemical', 'formula']
assert upload.total_calcs == 1
assert len(upload.calcs) == 1
with upload.entries_metadata() as entries:
assert_upload_files(upload.upload_id, entries, StagingUploadFiles, published=False)
assert_search_upload(entries, additional_keys, published=False)
def test_read_metadata_from_file(proc_infra, test_user, other_test_user):
upload = run_processing(
('test_upload', 'tests/data/proc/examples_with_metadata_file.zip'), test_user)
......
......@@ -68,7 +68,8 @@ parser_examples = [
('parser/fleur', 'tests/data/parsers/fleur/out'),
('parser/molcas', 'tests/data/parsers/molcas/test000.input.out'),
('parsers/qbox', 'tests/data/parsers/qbox/01_h2ogs.r'),
('parser/onetep', 'tests/data/parsers/onetep/single_point_2.out')
('parser/onetep', 'tests/data/parsers/onetep/single_point_2.out'),
('parsers/archive', 'tests/data/parsers/archive.json')
]
# We need to remove some cases with external mainfiles, which might not exist
......@@ -80,7 +81,7 @@ for parser, mainfile in parser_examples:
parser_examples = fixed_parser_examples
correct_num_output_files = 114
correct_num_output_files = 115
class TestBackend(object):
......@@ -253,9 +254,10 @@ def assert_parser_dir_unchanged(previous_wd, current_wd):
def run_parser(parser_name, mainfile):
parser = parser_dict[parser_name]