Commit 205aab5a authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Adapt new experimental metainfo with spectra parser.

parent 6ef0bede
Pipeline #99987 passed with stages
in 31 minutes and 34 seconds
Subproject commit 70de5df39103fce7a6b555fe5e1ff7d3bb28fcfd Subproject commit 0f87c8b72c43c22239bae5c9cc628460ebc9411a
Subproject commit 3f52d75a63bb24f8b0254a305f6422791dfaa7a1 Subproject commit 7446531042b7a468ca12d2902dd1019ea1fab522
Subproject commit 725217b401a130b34260f29334e353c9548cd1fa Subproject commit 39ba3392d3f16d6d1e85a8363e34880530b380b0
...@@ -49,13 +49,13 @@ class EMSMetadata(MSection): ...@@ -49,13 +49,13 @@ class EMSMetadata(MSection):
data_type = Quantity(type=str, a_search=Search()) data_type = Quantity(type=str, a_search=Search())
probing_method = Quantity(type=str, a_search=Search()) probing_method = Quantity(type=str, a_search=Search())
# data metadata # origin metadata
repository_name = Quantity(type=str, a_search=Search()) repository_name = Quantity(type=str, a_search=Search())
repository_url = Quantity(type=str, a_search=Search()) repository_url = Quantity(type=str, a_search=Search())
entry_repository_url = Quantity(type=str, a_search=Search()) entry_repository_url = Quantity(type=str, a_search=Search())
preview_url = Quantity(type=str, a_search=Search()) preview_url = Quantity(type=str, a_search=Search())
# TODO move # TODO move to more a general metadata section
quantities = Quantity(type=str, shape=['0..*'], default=[], a_search=Search()) quantities = Quantity(type=str, shape=['0..*'], default=[], a_search=Search())
group_hash = Quantity(type=str, a_search=Search()) group_hash = Quantity(type=str, a_search=Search())
...@@ -68,23 +68,34 @@ class EMSMetadata(MSection): ...@@ -68,23 +68,34 @@ class EMSMetadata(MSection):
entry = self.m_parent entry = self.m_parent
root_section = entry_archive.section_measurement[0] root_section = entry_archive.section_measurement[0]
entry.formula = root_section.section_metadata.section_sample.formula
atoms = root_section.section_metadata.section_sample.elements
if atoms is None: sample = root_section.section_metadata.section_sample
entry.atoms = [] entry.formula = config.services.unavailable_value
else: if sample:
if hasattr(atoms, 'tolist'): # TODO deal with multiple materials
atoms = atoms.tolist() material = sample.section_material[0] if len(sample.section_material) > 0 else None
entry.n_atoms = len(atoms) if material:
entry.formula = _unavailable(material.formula)
atoms = list(set(atoms)) atoms = material.elements
atoms.sort()
entry.atoms = atoms if atoms is None:
entry.atoms = []
# self.chemical = _unavailable(root_section.section_sample.section_material.chemical_name) else:
# self.sample_microstructure = _unavailable(root_section.section_sample.sample_microstructure) if hasattr(atoms, 'tolist'):
# self.sample_constituents = _unavailable(root_section.section_sample.sample_constituents) atoms = atoms.tolist()
entry.n_atoms = len(atoms)
atoms = list(set(atoms))
atoms.sort()
entry.atoms = atoms
if material.name:
self.chemical = _unavailable(material.name)
else:
self.chemical = _unavailable(material.formula)
self.sample_microstructure = _unavailable(sample.sample_microstructure)
self.sample_constituents = _unavailable(sample.sample_constituents)
self.experiment_summary = root_section.section_metadata.section_experiment.notes self.experiment_summary = root_section.section_metadata.section_experiment.notes
location = root_section.section_metadata.section_experiment.experiment_location location = root_section.section_metadata.section_experiment.experiment_location
...@@ -107,9 +118,9 @@ class EMSMetadata(MSection): ...@@ -107,9 +118,9 @@ class EMSMetadata(MSection):
# self.probing_method = _unavailable(root_section.section_method.probing_method) # self.probing_method = _unavailable(root_section.section_method.probing_method)
self.repository_name = _unavailable(root_section.section_metadata.section_origin.repository_name) self.repository_name = _unavailable(root_section.section_metadata.section_origin.repository_name)
self.repository_url = _unavailable(root_section.section_metadata.section_origin.repository_url) self.repository_url = root_section.section_metadata.section_origin.repository_url
self.preview_url = _unavailable(root_section.section_metadata.section_origin.preview_url) self.preview_url = root_section.section_metadata.section_origin.preview_url
self.entry_repository_url = _unavailable(root_section.section_metadata.section_origin.entry_repository_url) self.entry_repository_url = root_section.section_metadata.section_origin.entry_repository_url
self.group_hash = utils.hash( self.group_hash = utils.hash(
entry.formula, entry.formula,
......
...@@ -19,8 +19,7 @@ ...@@ -19,8 +19,7 @@
import numpy as np import numpy as np
from nomad.metainfo import ( from nomad.metainfo import (
MSection, MCategory, Package, Quantity, SubSection, MSection, MCategory, Package, Quantity, SubSection, Datetime)
Datetime, JSON)
m_package = Package(name='experimental_common') m_package = Package(name='experimental_common')
...@@ -31,6 +30,10 @@ class UserProvided(MCategory): ...@@ -31,6 +30,10 @@ class UserProvided(MCategory):
class SectionWithNotes(MSection): class SectionWithNotes(MSection):
'''
A common base-class for sections that should contain optional user provided
notes.
'''
notes = Quantity(type=str, categories=[UserProvided]) notes = Quantity(type=str, categories=[UserProvided])
...@@ -57,44 +60,46 @@ class DeviceSettings(MSection): ...@@ -57,44 +60,46 @@ class DeviceSettings(MSection):
class SampleMaterial(MSection): class SampleMaterial(MSection):
''' This section describes a sample's material. ''' ''' This section describes a sample's material. '''
chemical_formula = Quantity( elements = Quantity(
type=str, description='The chemical formula that describes the sample.') type=str, shape=['*'],
description='A list of element symbols for chemical elements in the material.')
chemical_name = Quantity(
type=str, description='The chemical name that describes the sample.')
atom_labels = Quantity( formula = Quantity(
type=str, shape=['number_of_elements'], type=str, description='The chemical formula that describes the material.')
description='Atom labels for distinct elements in the sample.')
number_of_elements = Quantity( name = Quantity(
type=int, derived=lambda m: len(m.atom_labels) if m.atom_labels else 0, type=str, description='The name that describes the material.')
description='Number of distinct chemical elements in the sample.')
space_group = Quantity( space_group_number = Quantity(
type=int, description='Space group of the sample compound (if crystalline).') type=int, description='Space group of the material (if crystalline).')
class Sample(SectionWithNotes): class Sample(SectionWithNotes):
sample_id = Quantity( sample_id = Quantity(
type=str, description='Identification number or signatures of the sample used.') type=str, description='Identification number or signatures of the sample used.')
sample_title = Quantity(type=str)
formula = Quantity(type=str) sample_name = Quantity(
elements = Quantity(type=str, shape=['*']) type=str, description='A human readable free text name for the sample.')
spectrum_region = Quantity(type=str, shape=[])
sample_description = Quantity( sample_description = Quantity(
type=str, description='Description of the sample used in the experiment.') type=str, description='A description of the sample.')
sample_state = Quantity( sample_state = Quantity(
type=str, description='The physical state of the sample.') type=str, description='The physical state of the sample.')
sample_temperature = Quantity( sample_temperature = Quantity(
type=np.dtype(np.float64), unit='kelvin', type=np.dtype(np.float64), unit='kelvin',
description='The temperature of the sample during the experiment.') description='The temperature of the sample during the experiment.')
sample_microstructure = Quantity( sample_microstructure = Quantity(
type=str, description='The sample microstructure.') type=str, description='The sample microstructure.')
sample_constituents = Quantity( sample_constituents = Quantity(
type=str, description='The constituents.') type=str, description='The constituents.')
section_material = SubSection(sub_section=SampleMaterial) spectrum_region = Quantity(type=str, shape=[])
section_material = SubSection(sub_section=SampleMaterial, repeats=True)
class ExperimentLocation(MSection): class ExperimentLocation(MSection):
......
This diff is collapsed.
...@@ -62,13 +62,14 @@ parser_exceptions = { ...@@ -62,13 +62,14 @@ parser_exceptions = {
'parsers/dl-poly': ['formula', 'dft.basis_set', 'dft.xc_functional', 'dft.system'], 'parsers/dl-poly': ['formula', 'dft.basis_set', 'dft.xc_functional', 'dft.system'],
'parsers/lib-atoms': ['dft.basis_set', 'dft.xc_functional'], 'parsers/lib-atoms': ['dft.basis_set', 'dft.xc_functional'],
'parsers/phonopy': ['dft.basis_set', 'dft.xc_functional'], 'parsers/phonopy': ['dft.basis_set', 'dft.xc_functional'],
'parsers/gamess': ['formula', 'dft.system', 'dft.xc_functional'], 'parsers/gamess': ['formula', 'atoms', 'dft.system', 'dft.xc_functional'],
'parsers/gulp': ['formula', 'dft.xc_functional', 'dft.system', 'dft.basis_set'], 'parsers/gulp': ['formula', 'atoms', 'dft.xc_functional', 'dft.system', 'dft.basis_set'],
'parsers/elastic': ['dft.basis_set', 'dft.xc_functional', 'dft.system'], 'parsers/elastic': ['dft.basis_set', 'dft.xc_functional', 'dft.system'],
'parsers/dmol': ['dft.system'], 'parsers/dmol': ['dft.system'],
'parsers/band': ['dft.system'], 'parsers/band': ['dft.system'],
'parsers/qbox': ['dft.xc_functional'], 'parsers/qbox': ['dft.xc_functional'],
'parser/onetep': ['formula', 'dft.basis_set', 'dft.xc_functional', 'dft.system'] 'parser/onetep': ['formula', 'atoms', 'dft.basis_set', 'dft.xc_functional', 'dft.system'],
'parsers/xps': ['formula', 'atoms']
} }
''' '''
Keys that the normalizer for certain parsers might not produce. In an ideal world this Keys that the normalizer for certain parsers might not produce. In an ideal world this
...@@ -84,8 +85,14 @@ def assert_normalized(entry_archive: datamodel.EntryArchive): ...@@ -84,8 +85,14 @@ def assert_normalized(entry_archive: datamodel.EntryArchive):
metadata = entry_archive.section_metadata metadata = entry_archive.section_metadata
metadata.apply_domain_metadata(entry_archive) metadata.apply_domain_metadata(entry_archive)
assert metadata.formula is not None parser_name = metadata.parser_name
assert len(metadata.atoms) is not None exceptions = parser_exceptions.get(parser_name, [])
assert metadata.atoms is not None
if 'formula' not in exceptions:
assert metadata.formula is not None and metadata.formula != config.services.unavailable_value
if 'atoms' not in exceptions:
assert len(metadata.atoms) > 0
if metadata.domain == 'dft': if metadata.domain == 'dft':
assert metadata.dft.code_name is not None assert metadata.dft.code_name is not None
...@@ -94,12 +101,6 @@ def assert_normalized(entry_archive: datamodel.EntryArchive): ...@@ -94,12 +101,6 @@ def assert_normalized(entry_archive: datamodel.EntryArchive):
assert metadata.dft.xc_functional is not None assert metadata.dft.xc_functional is not None
assert metadata.dft.system is not None assert metadata.dft.system is not None
parser_name = metadata.parser_name
exceptions = parser_exceptions.get(parser_name, [])
if metadata.formula != config.services.unavailable_value:
assert len(metadata.atoms) > 0
for key in calc_metadata_keys: for key in calc_metadata_keys:
if key in exceptions: if key in exceptions:
continue continue
......
...@@ -33,8 +33,9 @@ parser_examples = [ ...@@ -33,8 +33,9 @@ parser_examples = [
('parsers/random', 'test/data/parsers/random_0'), ('parsers/random', 'test/data/parsers/random_0'),
('parsers/template', 'tests/data/parsers/template.json'), ('parsers/template', 'tests/data/parsers/template.json'),
('parsers/eels', 'tests/data/parsers/eels.json'), ('parsers/eels', 'tests/data/parsers/eels.json'),
('parsers/aptfim', 'tests/data/parsers/aptfim.aptfim'), ('parsers/xps', 'tests/data/parsers/xps.json'),
('parsers/mpes', 'tests/data/parsers/mpes.meta'), # ('parsers/aptfim', 'tests/data/parsers/aptfim.aptfim'),
# ('parsers/mpes', 'tests/data/parsers/mpes.meta'),
('parsers/exciting', 'tests/data/parsers/exciting/Ag/INFO.OUT'), ('parsers/exciting', 'tests/data/parsers/exciting/Ag/INFO.OUT'),
('parsers/exciting', 'tests/data/parsers/exciting/GW/INFO.OUT'), ('parsers/exciting', 'tests/data/parsers/exciting/GW/INFO.OUT'),
('parsers/exciting', 'tests/data/parsers/exciting/nitrogen/INFO.OUT_nitrogen'), ('parsers/exciting', 'tests/data/parsers/exciting/nitrogen/INFO.OUT_nitrogen'),
...@@ -86,7 +87,7 @@ for parser, mainfile in parser_examples: ...@@ -86,7 +87,7 @@ for parser, mainfile in parser_examples:
parser_examples = fixed_parser_examples parser_examples = fixed_parser_examples
correct_num_output_files = 117 correct_num_output_files = 116
class TestBackend(object): class TestBackend(object):
......
...@@ -565,10 +565,9 @@ def test_malicious_parser_task_failure(proc_infra, failure, test_user): ...@@ -565,10 +565,9 @@ def test_malicious_parser_task_failure(proc_infra, failure, test_user):
def test_ems_data(proc_infra, test_user): def test_ems_data(proc_infra, test_user):
upload = run_processing(('test_ems_upload', 'tests/data/proc/examples_ems.zip'), test_user) upload = run_processing(('test_ems_upload', 'tests/data/proc/examples_ems.zip'), test_user)
additional_keys = [ additional_keys = ['ems.method', 'ems.origin_time']
'ems.method', 'formula', 'n_atoms', 'atoms', 'ems.origin_time'] assert upload.total_calcs == 2
assert upload.total_calcs == 1 assert len(upload.calcs) == 2
assert len(upload.calcs) == 1
with upload.entries_metadata() as entries: with upload.entries_metadata() as entries:
assert_upload_files(upload.upload_id, entries, StagingUploadFiles, published=False) assert_upload_files(upload.upload_id, entries, StagingUploadFiles, published=False)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment