Merge branch 'master' of https://gitlab.mpcdf.mpg.de/mkuehbac/parser-aptfim

Updated README.md. Added metadata.yml. The lead branch is now master, inactive branches became tags

aptfimparser/__init__.py

@@ -19,113 +19,82 @@ import re
 import numpy as np
 from datetime import datetime
 
-from nomadcore.simple_parser import SimpleMatcher
-from nomadcore.baseclasses import ParserInterface, AbstractBaseParser
+from nomad.parsing.parser import FairdiParser
+from nomad.datamodel.metainfo.common_experimental import (
+    Experiment, Data, Method, Sample, Location, Material)
 
-from nomad.parsing import LocalBackend
 
+class APTFIMParser(FairdiParser):
+    def __init__(self):
+        super().__init__(
+            name='parsers/aptfim', code_name='mpes', code_homepage='https://github.com/mpes-kit/mpes',
+            domain='ems', mainfile_mime_re=r'(application/json)|(text/.*)', mainfile_name_re=(r'.*.aptfim')
+        )
 
-class APTFIMParserInterface(ParserInterface):
-    def get_metainfo_filename(self):
-        """
-        The parser specific metainfo. To include other metadata definitions, use
-        the 'dependencies' key to refer to other local nomadmetainfo.json files or
-        to nomadmetainfo.json files that are part of the general nomad-meta-info
-        submodule (i.e. ``dependencies/nomad-meta-info``).
-         """
-        return os.path.join(os.path.dirname(__file__), 'aptfim.nomadmetainfo.json')
-
-    def get_parser_info(self):
-        """ Basic info about parser used in archive data and logs. """
-        return {
-            'name': 'aptfimparser',
-            'version': '0.1.0'
-        }
-
-    def setup_version(self):
-        """ Can be used to call :func:`setup_main_parser` differently for different code versions. """
-        self.setup_main_parser(None)
-
-    def setup_main_parser(self, _):
-        """ Setup the actual parser (behind this interface) """
-        self.main_parser = APTFIMParser(self.parser_context)
-
-
-class APTFIMParser(AbstractBaseParser):
-
-    def parse(self, filepath):
-        backend = self.parser_context.super_backend
-
+    def parse(self, filepath, archive, logger=None):
         with open(filepath, 'rt') as f:
             data = json.load(f)
-            #print(data)
-
-        # # You need to open sections before you can add values or sub sections to it.
-        # # The returned 'gid' can be used to reference a specific section if multiple
-        # # sections of the same type are opened.
-        #test_gid = backend.openSection('experiment_cantext')
-        #backend.addValue('context_headxx', data.get('experiment_type'))
-        #backend.closeSection('experiment_context', test_gid)
 
-        root_gid = backend.openSection('section_experiment')
-        # # Values do not necessarily have to be read from the parsed file.
-        # # The backend will check the type of the given value agains the metadata definition.
-        # backend.addValue('experiment_time', int(datetime.strptime(data.get('date'), '%d.%M.%Y').timestamp()))
-        #
-        # # Read data .
-#        data_gid = backend.openSection('section_context')
-        #data_gid = backend.openSection('section_experiment')
-        # addValue
-        # first argument STRING IDENTIFIER IN OUTPUT JSON KEYWORDS from aptfim.nomadmetainfo.json (ie. the generated parser result JSON)
-        # second argument STRING IDENTIFIER IN INPUT JSON (ie. the small META DATA FILE TO THE DATASET
-        #backend.addValue('data_repository_name', data.get('data_repository_name'))
-        #backend.addValue('data_repository_url', data.get('data_repository_url'))
-        #backend.addValue('data_preview_url', data.get('data_preview_url'))
-#        backend.addValue('real_one', data.get('experiment_typpe'))
-#        backend.closeSection('section_context', data_gid)
+        experiment = archive.m_create(Experiment)
+        experiment.raw_metadata = data
 
         # Read general tool environment details
-        # general_gid = backend.openSection('section_experiment_general_parameters')
-        
-        backend.addValue('experiment_method', data.get('experiment_method'))
-        backend.addValue('experiment_location', data.get('experiment_location'))
-        backend.addValue('experiment_facility_institution', data.get('experiment_facility_institution'))
-        backend.addValue('experiment_tool_info', data.get('instrument_info')) ###test here the case that input.json keyword is different to output.json
-        
-#        backend.addValue('experiment_data_global_start', np.array(re.findall(r"[\w']+", data.get('experiment_data_global_start'))))  ####
-#        backend.addValue('experiment_data_global_end', np.array(re.findall(r"[\w']+", data.get('experiment_data_global_end'))))  ####
-#        backend.addValue('experiment_data_local_start', np.array(re.findall(r"[\w']+", data.get('experiment_data_local_start')))) ####
-        
-#        backend.addValue('experiment_operation_method', data.get('experiment_operation_method'))
-#        backend.addValue('experiment_imaging_method', data.get('experiment_imaging_method'))
+        location = experiment.m_create(Location)
+        location.address = data.get('experiment_location')
+        location.facility = data.get('experiment_facility_institution')
+        experiment.experiment_summary = '%s of %s.' % (
+            data.get('experiment_method').capitalize(), data.get('specimen_description'))
+
+        try:
+            experiment.experiment_time = datetime.strptime(
+                data.get('experiment_date_global_start'), '%d.%m.%Y %M:%H:%S')
+        except ValueError:
+            pass
+
+        try:
+            experiment.experiment_end_time = datetime.strptime(
+                data.get('experiment_date_global_end'), '%d.%m.%Y %M:%H:%S')
+        except ValueError:
+            pass
+
+        # Read data parameters
+        section_data = experiment.m_create(Data)
+        section_data.repository_name = data.get('data_repository_name')
+        section_data.entry_repository_url = data.get('data_repository_url')
+        section_data.repository_url = '/'.join(data.get('data_repository_url').split('/')[0:3])
+        preview_url = data.get('data_preview_url')
+        # TODO: This a little hack to correct the preview url and should be removed
+        # after urls are corrected
+        preview_url = '%s/files/%s' % tuple(preview_url.rsplit('/', 1))
+        section_data.preview_url = preview_url
+
+        # Read parameters related to method
+        method = experiment.m_create(Method)
+        method.data_type = 'image'
+        method.method_name = data.get('experiment_method')
+        method.method_abbreviation = 'APT/FIM'
+        method.probing_method = 'electric pulsing'
+        method.instrument_description = data.get('instrument_info')
+
+        method.measured_number_ions_evaporated = data.get('measured_number_ions_evaporated')
+        method.measured_detector_hit_pos = data.get('measured_detector_hit_pos') == 'yes'
+        method.measured_detector_hit_mult = data.get('measured_detector_hit_mult') == 'yes'
+        method.measured_detector_dead_pulses = data.get('measured_detector_dead_pulses') == 'yes'
+        method.measured_time_of_flight = data.get('measured_time_of_flight') == 'yes'
+        method.measured_standing_voltage = data.get('measured_standing_voltage') == 'yes'
+        method.measured_pulse_voltage = data.get('measured_pulse_voltage') == 'yes'
+        method.experiment_operation_method = data.get('experiment_operation_method') == 'yes'
+        method.experiment_imaging_method = data.get('experiment_imaging_method') == 'yes'
 
         # Read parameters related to sample
-#        backend.addValue('specimen_description', data.get('specimen_description'))
-#        backend.addValue('specimen_microstructure', data.get('specimen_microstructure'))
-#        backend.addValue('specimen_constitution', data.get('specimen_constitution'))
-        
-        #### parse chemical composition
-        
-        ### measured_pulse_voltage for instance should be a conditional read
-#        backend.addValue('measured_number_ions_evaporated', data.get('measured_number_ions_evaporated'))
-#        backend.addValue('measured_detector_hit_pos', data.get('measured_detector_hit_pos'))
-#        backend.addValue('measured_detector_hit_mult', data.get('measured_detector_hit_mult'))
-#        backend.addValue('measured_detector_dead_pulses', data.get('measured_detector_dead_pulses'))
-#       backend.addValue('measured_time_of_flight', data.get('measured_time_of_flight'))
-#        backend.addValue('measured_standing_voltage', data.get('measured_standing_voltage'))
-#        backend.addValue('measured_pulse_voltage', data.get('measured_pulse_voltage'))
-
-        
-        # To add arrays (vectors, matrices, etc.) use addArrayValues and provide a
-        # numpy array. The shape of the numpy array must match the shape defined in
-        # the respective metadata definition.
-
-
-        # Close sections in the reverse order
-        #backend.closeSection('section_experiment', data_gid)
-        #backend.closeSection('section_data', data_gid)
-        backend.closeSection('section_experiment', root_gid)
-        # backend.closeSection('section_experiment_general_parameters', general_gid)
-        # backend.closeSection('section_experiment_source_parameters', source_gid)
-        # backend.closeSection('section_experiment_detector_parameters', detector_gid)
-        # backend.closeSection('section_experiment_sample_parameters', sample_gid)
+        sample = experiment.m_create(Sample)
+        sample.sample_description = data.get('specimen_description')
+        sample.sample_microstructure = data.get('specimen_microstructure')
+        sample.sample_constituents = data.get('specimen_constitution')
+
+        material = sample.m_create(Material)
+        atom_labels = data.get('specimen_chemistry')
+        formula = ase.Atoms(atom_labels).get_chemical_formula()
+        material.atom_labels = np.array(atom_labels)
+        material.chemical_formula = formula
+        material.chemical_name = formula

aptfimparser/__main__.py

@@ -12,15 +12,10 @@
 #   limitations under the License.
 
 import sys
+import json
 
-from nomad.parsing import LocalBackend
-from aptfimparser import APTFIMParserInterface
+from aptfimparser import APTFIMParser
 
 if __name__ == "__main__":
-    # instantiate the parser via its interface with a LocalBackend
-    parser = APTFIMParserInterface(backend=LocalBackend)
-    # call the actual parsing with the given mainfile
-    parser.parse(sys.argv[1])
-    # print the results stored in the LocalBackend
-    parser.parser_context.super_backend.write_json(
-        sys.stdout, pretty=True, root_sections=['section_experiment'])
+    archive = APTFIMParser.main(sys.argv[1])
+    json.dump(archive.m_to_dict(), sys.stdout, indent=2)

aptfimparser/aptfim.nomadmetainfo.json

-{
-  "type": "nomad_meta_info_1_0",
-  "description": "Metadata for an atom probe tomography or field ion microscopy experiment.",
-  "dependencies":[
-    {
-      "metainfoPath":"general.nomadmetainfo.json"
-    },
-    {
-      "metainfoPath":"general.experimental.nomadmetainfo.json"
-    }
-  ],
-  "metaInfos": [
-    {
-      "description": "String identifier aka name of the repository where the raw data to the experiment is available",
-      "name": "data_repository_name",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": ""
-    },
-    {
-      "description": "URL of this repository",
-      "name": "data_repository_url",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": ""
-    },
-    {      "description": "Thumbnail image informing about the experiment",
-      "name": "data_preview_url",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": ""
-    },
-    {
-      "description": "Shape of the None/Null object",
-      "name": "none_shape",
-      "dtypeStr": "i",
-      "kindStr": "type_dimension",
-      "shape": [],
-      "superNames": ["section_experiment"]
-    },
-    {
-      "description": "Full name of the experimental method in use",
-      "name": "experiment_method",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": ""
-    },
-    {
-      "description": "Name of the city and country the experiment took place, format 'Country, City'",
-      "name": "experiment_location",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": ""
-    },
-    {
-      "description": "Name of the institution hosting the experimental facility",
-      "name": "experiment_facility_institution",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": ""
-    },
-    {
-      "description": "Name of the equipment, instrument with which the experiment was performed e.g. LEAP5000XS",
-      "name": "experiment_tool_info",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": ""
-    },
-    {
-      "description": "UTC start time of the experiment, format 'DD.MM.YYYY - HH.MM.SS'",
-      "name": "experiment_date_global_start",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": "C"
-    },
-    {
-      "description": "UTC end time of the experiment, format 'DD.MM.YYYY - HH.MM.SS'",
-      "name": "experiment_date_global_end",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": "C"
-    },
-    {
-      "description": "Local start time of the experiment, format 'DD.MM.YYYY - HH.MM.SS'",
-      "name": "experiment_date_local_start",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": "C"
-    },
-    {
-      "description": "Operation mode of the instrument (APT, FIM or combination)",
-      "name": "experiment_operation_method",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": ""
-    },
-    {
-      "description": "Pulsing method to enforce a controlled ion evaporation sequence",
-      "name": "experiment_imaging_method",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": ""
-    },
-    {
-      "description": "Sample description e.g. pure W wire samples trial 2",
-      "name": "specimen_description",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": ""
-    },
-    {
-      "description": "Number of elements (disjoint element names) expected",
-      "name": "number_of_elements",
-      "dtypeStr": "i",
-      "kindStr": "type_dimension",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": ""
-    },
-    {
-      "description": "List of periodic table names expected contained in dataset",
-      "name": "specimen_chemistry",
-      "dtypeStr": "C",
-      "shape": ["number_of_elements"],
-      "superNames": ["section_experiment"],
-      "units": ""
-    },
-    {
-      "description": "Qualitative type of specimen and microstructure analyzed (e.g. thin films, nano objects, single crystal, polycrystal)",
-      "name": "specimen_microstructure",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": ""
-    },
-    {
-      "description": "Qualitative information how many phases in the specimen",
-      "name": "specimen_constitution",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": ""
-    },
-	{
-      "description": "Number of ions successfully evaporated",
-      "name": "measured_number_ions_evaporated",
-      "dtypeStr": "i",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": "1"
-    },
-    {
-      "description": "Detector hit positions x and y",
-      "name": "measured_detector_hit_pos",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": "mm, mm"
-    },
-    {
-      "description": "Detector hit multiplicity",
-      "name": "measured_detector_hit_mult",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": "1"
-    },
-    {
-      "description": "Detector number of dead pulses",
-      "name": "measured_detector_dead_pulses",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": "1"
-    },
-    {
-      "description": "Raw ion time of flight",
-      "name": "measured_time_of_flight",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": "ns"
-    },
-    {
-      "description": "Standing voltage",
-      "name": "measured_standing_voltage",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": "V"
-    },
-    {
-      "description": "Pulse voltage",
-      "name": "measured_pulse_voltage",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": "V"
-    }
-  ]
-}

aptfimparser/metainfo/__init__.py

+import sys
+from nomad.metainfo import Environment
+from nomad.metainfo.legacy import LegacyMetainfoEnvironment
+import aptfimparser.metainfo.aptfim
+import nomad.datamodel.metainfo.general
+import nomad.datamodel.metainfo.common_experimental
+
+m_env = LegacyMetainfoEnvironment()
+m_env.m_add_sub_section(Environment.packages, sys.modules['aptfimparser.metainfo.aptfim'].m_package)  # type: ignore
+m_env.m_add_sub_section(Environment.packages, sys.modules['nomad.datamodel.metainfo.general'].m_package)  # type: ignore
+m_env.m_add_sub_section(Environment.packages, sys.modules['nomad.datamodel.metainfo.common_experimental'].m_package)  # type: ignore

aptfimparser/metainfo/aptfim.py

+from nomad.metainfo import Package, Quantity, Section
+from nomad.datamodel.metainfo import common_experimental
+
+m_package = Package(name='aptfim')
+
+
+class Method(common_experimental.Method):
+
+    m_def = Section(validate=False, extends_base_section=True)
+
+    experiment_operation_method = Quantity(
+        type=str,
+        description='Operation mode of the instrument (APT, FIM or combination)')
+
+    experiment_imaging_method = Quantity(
+        type=str,
+        description='Pulsing method to enforce a controlled ion evaporation sequence')
+
+    number_ions_evaporated = Quantity(
+        type=int,
+        description='Number of ions successfully evaporated')
+
+    measured_detector_hit_pos = Quantity(
+        type=bool,
+        description='Detector hit positions x and y was measured')
+
+    measured_detector_hit_mult = Quantity(
+        type=bool,
+        description='Detector hit multiplicity was measured')
+
+    measured_detector_dead_pulses = Quantity(
+        type=bool,
+        description='Detector number of dead pulses was measured')
+
+    measured_time_of_flight = Quantity(
+        type=bool,
+        description='Raw ion time of flight was measured')
+
+    measured_standing_voltage = Quantity(
+        type=bool,
+        description='Standing voltage was measured')
+
+    measured_pulse_voltage = Quantity(
+        type=bool,
+        description='Pulse voltage was measured')
+
+
+m_package.__init_metainfo__()

questions/aptfim.nomadmetainfo.keyerror.json

-{
-  "type": "nomad_meta_info_1_0",
-  "description": "Metadata for an atom probe tomography or field ion microscopy experiment.",
-  "dependencies":[
-    {
-      "metainfoPath":"general.nomadmetainfo.json"
-    },
-    {
-      "metainfoPath":"general.experimental.nomadmetainfo.json"
-    }
-  ],
-  "metaInfos": [
-    {
-      "description": "String identifier aka name of the repository where the raw data to the experiment is available",
-      "name": "experiment_typpe",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_context"]
-    },
-    {
-      "description": "String identifier aka name of the repository where the raw data to the experiment is available",
-      "name": "data_repository_name",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": ""
-    },
-    {
-      "description": "URL of this repository",
-      "name": "where",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": ""
-    },
-    {
-      "description": "Thumbnail image informing about the experiment",
-      "name": "data_preview_url",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": ""
-    },
-    {
-      "description": "Shape of the None/Null object",
-      "name": "none_shape",
-      "dtypeStr": "i",
-      "kindStr": "type_dimension",
-      "shape": [],
-      "superNames": ["section_experiment"]
-    },
-    {
-      "description": "Full name of the experimental method in use",
-      "name": "experiment_method",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": ""
-    },
-    {
-      "description": "Name of the city and country the experiment took place, format 'Country, City'",
-      "name": "experiment_location",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": ""
-    },
-    {
-      "description": "Name of the institution hosting the experimental facility",
-      "name": "experiment_facility_institution",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": ""
-    },
-    {
-      "description": "Name of the equipment, instrument with which the experiment was performed e.g. LEAP5000XS",
-      "name": "experiment_tool_info",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": ""
-    },
-    {
-      "description": "UTC start time of the experiment, format 'DD.MM.YYYY - HH.MM.SS'",
-      "name": "experiment_date_global_start",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": "C"
-    },
-    {
-      "description": "UTC end time of the experiment, format 'DD.MM.YYYY - HH.MM.SS'",
-      "name": "experiment_date_global_end",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": "C"
-    },
-    {
-      "description": "Local start time of the experiment, format 'DD.MM.YYYY - HH.MM.SS'",
-      "name": "experiment_date_local_start",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": "C"
-    },
-    {
-      "description": "Operation mode of the instrument (APT, FIM or combination)",
-      "name": "experiment_operation_method",
-      "dtypeStr": "C",
-      "shape": [],
-      "superNames": ["section_experiment"],
-      "units": ""
-    },
-    {
-      "description": "Pulsing method to enforce a controlled ion evaporation sequence",
-      "name": "experiment_imaging_method",
-      "dtypeStr": "C",