Adapt new experimental metainfo with spectra parser.

eelsdb @ 0f87c8b7

-Subproject commit 70de5df39103fce7a6b555fe5e1ff7d3bb28fcfd
+Subproject commit 0f87c8b72c43c22239bae5c9cc628460ebc9411a

xps @ 74465310

-Subproject commit 3f52d75a63bb24f8b0254a305f6422791dfaa7a1
+Subproject commit 7446531042b7a468ca12d2902dd1019ea1fab522

python_common @ 39ba3392

-Subproject commit 725217b401a130b34260f29334e353c9548cd1fa
+Subproject commit 39ba3392d3f16d6d1e85a8363e34880530b380b0

nomad/datamodel/ems.py

@@ -49,13 +49,13 @@ class EMSMetadata(MSection):
     data_type = Quantity(type=str, a_search=Search())
     probing_method = Quantity(type=str, a_search=Search())
 
-    # data metadata
+    # origin metadata
     repository_name = Quantity(type=str, a_search=Search())
     repository_url = Quantity(type=str, a_search=Search())
     entry_repository_url = Quantity(type=str, a_search=Search())
     preview_url = Quantity(type=str, a_search=Search())
 
-    # TODO move
+    # TODO move to more a general metadata section
     quantities = Quantity(type=str, shape=['0..*'], default=[], a_search=Search())
     group_hash = Quantity(type=str, a_search=Search())
 
@@ -68,23 +68,34 @@ class EMSMetadata(MSection):
         entry = self.m_parent
 
         root_section = entry_archive.section_measurement[0]
-        entry.formula = root_section.section_metadata.section_sample.formula
-        atoms = root_section.section_metadata.section_sample.elements
 
-        if atoms is None:
-            entry.atoms = []
-        else:
-            if hasattr(atoms, 'tolist'):
-                atoms = atoms.tolist()
-            entry.n_atoms = len(atoms)
-
-            atoms = list(set(atoms))
-            atoms.sort()
-            entry.atoms = atoms
-
-        # self.chemical = _unavailable(root_section.section_sample.section_material.chemical_name)
-        # self.sample_microstructure = _unavailable(root_section.section_sample.sample_microstructure)
-        # self.sample_constituents = _unavailable(root_section.section_sample.sample_constituents)
+        sample = root_section.section_metadata.section_sample
+        entry.formula = config.services.unavailable_value
+        if sample:
+            # TODO deal with multiple materials
+            material = sample.section_material[0] if len(sample.section_material) > 0 else None
+            if material:
+                entry.formula = _unavailable(material.formula)
+                atoms = material.elements
+
+                if atoms is None:
+                    entry.atoms = []
+                else:
+                    if hasattr(atoms, 'tolist'):
+                        atoms = atoms.tolist()
+                    entry.n_atoms = len(atoms)
+
+                    atoms = list(set(atoms))
+                    atoms.sort()
+                    entry.atoms = atoms
+
+                if material.name:
+                    self.chemical = _unavailable(material.name)
+                else:
+                    self.chemical = _unavailable(material.formula)
+
+            self.sample_microstructure = _unavailable(sample.sample_microstructure)
+            self.sample_constituents = _unavailable(sample.sample_constituents)
 
         self.experiment_summary = root_section.section_metadata.section_experiment.notes
         location = root_section.section_metadata.section_experiment.experiment_location
@@ -107,9 +118,9 @@ class EMSMetadata(MSection):
         # self.probing_method = _unavailable(root_section.section_method.probing_method)
 
         self.repository_name = _unavailable(root_section.section_metadata.section_origin.repository_name)
-        self.repository_url = _unavailable(root_section.section_metadata.section_origin.repository_url)
-        self.preview_url = _unavailable(root_section.section_metadata.section_origin.preview_url)
-        self.entry_repository_url = _unavailable(root_section.section_metadata.section_origin.entry_repository_url)
+        self.repository_url = root_section.section_metadata.section_origin.repository_url
+        self.preview_url = root_section.section_metadata.section_origin.preview_url
+        self.entry_repository_url = root_section.section_metadata.section_origin.entry_repository_url
 
         self.group_hash = utils.hash(
             entry.formula,

nomad/datamodel/metainfo/common_experimental.py

@@ -19,8 +19,7 @@
 import numpy as np
 
 from nomad.metainfo import (
-    MSection, MCategory, Package, Quantity, SubSection,
-    Datetime, JSON)
+    MSection, MCategory, Package, Quantity, SubSection, Datetime)
 
 
 m_package = Package(name='experimental_common')
@@ -31,6 +30,10 @@ class UserProvided(MCategory):
 
 
 class SectionWithNotes(MSection):
+    '''
+    A common base-class for sections that should contain optional user provided
+    notes.
+    '''
     notes = Quantity(type=str, categories=[UserProvided])
 
 
@@ -57,44 +60,46 @@ class DeviceSettings(MSection):
 
 class SampleMaterial(MSection):
     ''' This section describes a sample's material. '''
-    chemical_formula = Quantity(
-        type=str, description='The chemical formula that describes the sample.')
-
-    chemical_name = Quantity(
-        type=str, description='The chemical name that describes the sample.')
+    elements = Quantity(
+        type=str, shape=['*'],
+        description='A list of element symbols for chemical elements in the material.')
 
-    atom_labels = Quantity(
-        type=str, shape=['number_of_elements'],
-        description='Atom labels for distinct elements in the sample.')
+    formula = Quantity(
+        type=str, description='The chemical formula that describes the material.')
 
-    number_of_elements = Quantity(
-        type=int, derived=lambda m: len(m.atom_labels) if m.atom_labels else 0,
-        description='Number of distinct chemical elements in the sample.')
+    name = Quantity(
+        type=str, description='The name that describes the material.')
 
-    space_group = Quantity(
-        type=int, description='Space group of the sample compound (if crystalline).')
+    space_group_number = Quantity(
+        type=int, description='Space group of the material (if crystalline).')
 
 
 class Sample(SectionWithNotes):
     sample_id = Quantity(
         type=str, description='Identification number or signatures of the sample used.')
-    sample_title = Quantity(type=str)
-    formula = Quantity(type=str)
-    elements = Quantity(type=str, shape=['*'])
-    spectrum_region = Quantity(type=str, shape=[])
+
+    sample_name = Quantity(
+        type=str, description='A human readable free text name for the sample.')
+
     sample_description = Quantity(
-        type=str, description='Description of the sample used in the experiment.')
+        type=str, description='A description of the sample.')
+
     sample_state = Quantity(
         type=str, description='The physical state of the sample.')
+
     sample_temperature = Quantity(
         type=np.dtype(np.float64), unit='kelvin',
         description='The temperature of the sample during the experiment.')
+
     sample_microstructure = Quantity(
         type=str, description='The sample microstructure.')
+
     sample_constituents = Quantity(
         type=str, description='The constituents.')
 
-    section_material = SubSection(sub_section=SampleMaterial)
+    spectrum_region = Quantity(type=str, shape=[])
+
+    section_material = SubSection(sub_section=SampleMaterial, repeats=True)
 
 
 class ExperimentLocation(MSection):

tests/normalizing/test_system.py

@@ -62,13 +62,14 @@ parser_exceptions = {
     'parsers/dl-poly': ['formula', 'dft.basis_set', 'dft.xc_functional', 'dft.system'],
     'parsers/lib-atoms': ['dft.basis_set', 'dft.xc_functional'],
     'parsers/phonopy': ['dft.basis_set', 'dft.xc_functional'],
-    'parsers/gamess': ['formula', 'dft.system', 'dft.xc_functional'],
-    'parsers/gulp': ['formula', 'dft.xc_functional', 'dft.system', 'dft.basis_set'],
+    'parsers/gamess': ['formula', 'atoms', 'dft.system', 'dft.xc_functional'],
+    'parsers/gulp': ['formula', 'atoms', 'dft.xc_functional', 'dft.system', 'dft.basis_set'],
     'parsers/elastic': ['dft.basis_set', 'dft.xc_functional', 'dft.system'],
     'parsers/dmol': ['dft.system'],
     'parsers/band': ['dft.system'],
     'parsers/qbox': ['dft.xc_functional'],
-    'parser/onetep': ['formula', 'dft.basis_set', 'dft.xc_functional', 'dft.system']
+    'parser/onetep': ['formula', 'atoms', 'dft.basis_set', 'dft.xc_functional', 'dft.system'],
+    'parsers/xps': ['formula', 'atoms']
 }
 '''
 Keys that the normalizer for certain parsers might not produce. In an ideal world this
@@ -84,8 +85,14 @@ def assert_normalized(entry_archive: datamodel.EntryArchive):
 
     metadata = entry_archive.section_metadata
     metadata.apply_domain_metadata(entry_archive)
-    assert metadata.formula is not None
-    assert len(metadata.atoms) is not None
+    parser_name = metadata.parser_name
+    exceptions = parser_exceptions.get(parser_name, [])
+
+    assert metadata.atoms is not None
+    if 'formula' not in exceptions:
+        assert metadata.formula is not None and metadata.formula != config.services.unavailable_value
+    if 'atoms' not in exceptions:
+        assert len(metadata.atoms) > 0
 
     if metadata.domain == 'dft':
         assert metadata.dft.code_name is not None
@@ -94,12 +101,6 @@ def assert_normalized(entry_archive: datamodel.EntryArchive):
         assert metadata.dft.xc_functional is not None
         assert metadata.dft.system is not None
 
-    parser_name = metadata.parser_name
-    exceptions = parser_exceptions.get(parser_name, [])
-
-    if metadata.formula != config.services.unavailable_value:
-        assert len(metadata.atoms) > 0
-
     for key in calc_metadata_keys:
         if key in exceptions:
             continue

tests/parsing/test_parsing.py

@@ -33,8 +33,9 @@ parser_examples = [
     ('parsers/random', 'test/data/parsers/random_0'),
     ('parsers/template', 'tests/data/parsers/template.json'),
     ('parsers/eels', 'tests/data/parsers/eels.json'),
-    ('parsers/aptfim', 'tests/data/parsers/aptfim.aptfim'),
-    ('parsers/mpes', 'tests/data/parsers/mpes.meta'),
+    ('parsers/xps', 'tests/data/parsers/xps.json'),
+    # ('parsers/aptfim', 'tests/data/parsers/aptfim.aptfim'),
+    # ('parsers/mpes', 'tests/data/parsers/mpes.meta'),
     ('parsers/exciting', 'tests/data/parsers/exciting/Ag/INFO.OUT'),
     ('parsers/exciting', 'tests/data/parsers/exciting/GW/INFO.OUT'),
     ('parsers/exciting', 'tests/data/parsers/exciting/nitrogen/INFO.OUT_nitrogen'),
@@ -86,7 +87,7 @@ for parser, mainfile in parser_examples:
 parser_examples = fixed_parser_examples
 
 
-correct_num_output_files = 117
+correct_num_output_files = 116
 
 
 class TestBackend(object):

tests/processing/test_data.py

@@ -565,10 +565,9 @@ def test_malicious_parser_task_failure(proc_infra, failure, test_user):
 def test_ems_data(proc_infra, test_user):
     upload = run_processing(('test_ems_upload', 'tests/data/proc/examples_ems.zip'), test_user)
 
-    additional_keys = [
-        'ems.method', 'formula', 'n_atoms', 'atoms', 'ems.origin_time']
-    assert upload.total_calcs == 1
-    assert len(upload.calcs) == 1
+    additional_keys = ['ems.method', 'ems.origin_time']
+    assert upload.total_calcs == 2
+    assert len(upload.calcs) == 2
 
     with upload.entries_metadata() as entries:
         assert_upload_files(upload.upload_id, entries, StagingUploadFiles, published=False)