Initial commit

aptfimparser/__init__.py

+# Copyright 2016-2019 Markus Scheidgen, Markus Kühbach
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+import sys
+import os.path
+import json
+import ase
+import re
+import numpy as np
+from datetime import datetime
+
+from nomadcore.simple_parser import SimpleMatcher
+from nomadcore.baseclasses import ParserInterface, AbstractBaseParser
+
+from nomad.parsing import LocalBackend
+
+
+class APTFIMParserInterface(ParserInterface):
+    def get_metainfo_filename(self):
+        """
+        The parser specific metainfo. To include other metadata definitions, use
+        the 'dependencies' key to refer to other local nomadmetainfo.json files or
+        to nomadmetainfo.json files that are part of the general nomad-meta-info
+        submodule (i.e. ``dependencies/nomad-meta-info``).
+         """
+        return os.path.join(os.path.dirname(__file__), 'aptfim.nomadmetainfo.json')
+
+    def get_parser_info(self):
+        """ Basic info about parser used in archive data and logs. """
+        return {
+            'name': 'aptfimparser',
+            'version': '0.1.0'
+        }
+
+    def setup_version(self):
+        """ Can be used to call :func:`setup_main_parser` differently for different code versions. """
+        self.setup_main_parser(None)
+
+    def setup_main_parser(self, _):
+        """ Setup the actual parser (behind this interface) """
+        self.main_parser = APTFIMParser(self.parser_context)
+
+
+class APTFIMParser(AbstractBaseParser):
+
+    def parse(self, filepath):
+        backend = self.parser_context.super_backend
+
+        with open(filepath, 'rt') as f:
+            data = json.load(f)
+            #print(data)
+
+        # # You need to open sections before you can add values or sub sections to it.
+        # # The returned 'gid' can be used to reference a specific section if multiple
+        # # sections of the same type are opened.
+        #test_gid = backend.openSection('experiment_cantext')
+        #backend.addValue('context_headxx', data.get('experiment_type'))
+        #backend.closeSection('experiment_context', test_gid)
+
+        root_gid = backend.openSection('section_experiment')
+        # # Values do not necessarily have to be read from the parsed file.
+        # # The backend will check the type of the given value agains the metadata definition.
+        # backend.addValue('experiment_time', int(datetime.strptime(data.get('date'), '%d.%M.%Y').timestamp()))
+        #
+        # # Read data .
+#        data_gid = backend.openSection('section_context')
+        #data_gid = backend.openSection('section_experiment')
+        # addValue
+        # first argument STRING IDENTIFIER IN OUTPUT JSON KEYWORDS from aptfim.nomadmetainfo.json (ie. the generated parser result JSON)
+        # second argument STRING IDENTIFIER IN INPUT JSON (ie. the small META DATA FILE TO THE DATASET
+        #backend.addValue('data_repository_name', data.get('data_repository_name'))
+        #backend.addValue('data_repository_url', data.get('data_repository_url'))
+        #backend.addValue('data_preview_url', data.get('data_preview_url'))
+#        backend.addValue('real_one', data.get('experiment_typpe'))
+#        backend.closeSection('section_context', data_gid)
+
+        # Read general tool environment details
+        # general_gid = backend.openSection('section_experiment_general_parameters')
+        
+        backend.addValue('experiment_method', data.get('experiment_method'))
+        backend.addValue('experiment_location', data.get('experiment_location'))
+        backend.addValue('experiment_facility_institution', data.get('experiment_facility_institution'))
+        backend.addValue('experiment_tool_info', data.get('instrument_info')) ###test here the case that input.json keyword is different to output.json
+        
+#        backend.addValue('experiment_data_global_start', np.array(re.findall(r"[\w']+", data.get('experiment_data_global_start'))))  ####
+#        backend.addValue('experiment_data_global_end', np.array(re.findall(r"[\w']+", data.get('experiment_data_global_end'))))  ####
+#        backend.addValue('experiment_data_local_start', np.array(re.findall(r"[\w']+", data.get('experiment_data_local_start')))) ####
+        
+#        backend.addValue('experiment_operation_method', data.get('experiment_operation_method'))
+#        backend.addValue('experiment_imaging_method', data.get('experiment_imaging_method'))
+
+        # Read parameters related to sample
+#        backend.addValue('specimen_description', data.get('specimen_description'))
+#        backend.addValue('specimen_microstructure', data.get('specimen_microstructure'))
+#        backend.addValue('specimen_constitution', data.get('specimen_constitution'))
+        
+        #### parse chemical composition
+        
+        ### measured_pulse_voltage for instance should be a conditional read
+#        backend.addValue('measured_number_ions_evaporated', data.get('measured_number_ions_evaporated'))
+#        backend.addValue('measured_detector_hit_pos', data.get('measured_detector_hit_pos'))
+#        backend.addValue('measured_detector_hit_mult', data.get('measured_detector_hit_mult'))
+#        backend.addValue('measured_detector_dead_pulses', data.get('measured_detector_dead_pulses'))
+#       backend.addValue('measured_time_of_flight', data.get('measured_time_of_flight'))
+#        backend.addValue('measured_standing_voltage', data.get('measured_standing_voltage'))
+#        backend.addValue('measured_pulse_voltage', data.get('measured_pulse_voltage'))
+
+        
+        # To add arrays (vectors, matrices, etc.) use addArrayValues and provide a
+        # numpy array. The shape of the numpy array must match the shape defined in
+        # the respective metadata definition.
+
+
+        # Close sections in the reverse order
+        #backend.closeSection('section_experiment', data_gid)
+        #backend.closeSection('section_data', data_gid)
+        backend.closeSection('section_experiment', root_gid)
+        # backend.closeSection('section_experiment_general_parameters', general_gid)
+        # backend.closeSection('section_experiment_source_parameters', source_gid)
+        # backend.closeSection('section_experiment_detector_parameters', detector_gid)
+        # backend.closeSection('section_experiment_sample_parameters', sample_gid)

aptfimparser/__main__.py

+# Copyright 2016-2018 Markus Scheidgen
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+import sys
+
+from nomad.parsing import LocalBackend
+from aptfimparser import APTFIMParserInterface
+
+if __name__ == "__main__":
+    # instantiate the parser via its interface with a LocalBackend
+    parser = APTFIMParserInterface(backend=LocalBackend)
+    # call the actual parsing with the given mainfile
+    parser.parse(sys.argv[1])
+    # print the results stored in the LocalBackend
+    parser.parser_context.super_backend.write_json(
+        sys.stdout, pretty=True, root_sections=['section_experiment'])

aptfimparser/aptfim.nomadmetainfo.json

+{
+  "type": "nomad_meta_info_1_0",
+  "description": "Metadata for an atom probe tomography or field ion microscopy experiment.",
+  "dependencies":[
+    {
+      "metainfoPath":"general.nomadmetainfo.json"
+    },
+    {
+      "metainfoPath":"general.experimental.nomadmetainfo.json"
+    }
+  ],
+  "metaInfos": [
+    {
+      "description": "String identifier aka name of the repository where the raw data to the experiment is available",
+      "name": "data_repository_name",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": ""
+    },
+    {
+      "description": "URL of this repository",
+      "name": "data_repository_url",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": ""
+    },
+    {      "description": "Thumbnail image informing about the experiment",
+      "name": "data_preview_url",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": ""
+    },
+    {
+      "description": "Shape of the None/Null object",
+      "name": "none_shape",
+      "dtypeStr": "i",
+      "kindStr": "type_dimension",
+      "shape": [],
+      "superNames": ["section_experiment"]
+    },
+    {
+      "description": "Full name of the experimental method in use",
+      "name": "experiment_method",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": ""
+    },
+    {
+      "description": "Name of the city and country the experiment took place, format 'Country, City'",
+      "name": "experiment_location",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": ""
+    },
+    {
+      "description": "Name of the institution hosting the experimental facility",
+      "name": "experiment_facility_institution",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": ""
+    },
+    {
+      "description": "Name of the equipment, instrument with which the experiment was performed e.g. LEAP5000XS",
+      "name": "experiment_tool_info",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": ""
+    },
+    {
+      "description": "UTC start time of the experiment, format 'DD.MM.YYYY - HH.MM.SS'",
+      "name": "experiment_date_global_start",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": "C"
+    },
+    {
+      "description": "UTC end time of the experiment, format 'DD.MM.YYYY - HH.MM.SS'",
+      "name": "experiment_date_global_end",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": "C"
+    },
+    {
+      "description": "Local start time of the experiment, format 'DD.MM.YYYY - HH.MM.SS'",
+      "name": "experiment_date_local_start",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": "C"
+    },
+    {
+      "description": "Operation mode of the instrument (APT, FIM or combination)",
+      "name": "experiment_operation_method",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": ""
+    },
+    {
+      "description": "Pulsing method to enforce a controlled ion evaporation sequence",
+      "name": "experiment_imaging_method",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": ""
+    },
+    {
+      "description": "Sample description e.g. pure W wire samples trial 2",
+      "name": "specimen_description",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": ""
+    },
+    {
+      "description": "Number of elements (disjoint element names) expected",
+      "name": "number_of_elements",
+      "dtypeStr": "i",
+      "kindStr": "type_dimension",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": ""
+    },
+    {
+      "description": "List of periodic table names expected contained in dataset",
+      "name": "specimen_chemistry",
+      "dtypeStr": "C",
+      "shape": ["number_of_elements"],
+      "superNames": ["section_experiment"],
+      "units": ""
+    },
+    {
+      "description": "Qualitative type of specimen and microstructure analyzed (e.g. thin films, nano objects, single crystal, polycrystal)",
+      "name": "specimen_microstructure",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": ""
+    },
+    {
+      "description": "Qualitative information how many phases in the specimen",
+      "name": "specimen_constitution",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": ""
+    },
+	{
+      "description": "Number of ions successfully evaporated",
+      "name": "measured_number_ions_evaporated",
+      "dtypeStr": "i",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": "1"
+    },
+    {
+      "description": "Detector hit positions x and y",
+      "name": "measured_detector_hit_pos",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": "mm, mm"
+    },
+    {
+      "description": "Detector hit multiplicity",
+      "name": "measured_detector_hit_mult",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": "1"
+    },
+    {
+      "description": "Detector number of dead pulses",
+      "name": "measured_detector_dead_pulses",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": "1"
+    },
+    {
+      "description": "Raw ion time of flight",
+      "name": "measured_time_of_flight",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": "ns"
+    },
+    {
+      "description": "Standing voltage",
+      "name": "measured_standing_voltage",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": "V"
+    },
+    {
+      "description": "Pulse voltage",
+      "name": "measured_pulse_voltage",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": "V"
+    }
+  ]
+}

questions/aptfim.nomadmetainfo.keyerror.json

+{
+  "type": "nomad_meta_info_1_0",
+  "description": "Metadata for an atom probe tomography or field ion microscopy experiment.",
+  "dependencies":[
+    {
+      "metainfoPath":"general.nomadmetainfo.json"
+    },
+    {
+      "metainfoPath":"general.experimental.nomadmetainfo.json"
+    }
+  ],
+  "metaInfos": [
+    {
+      "description": "String identifier aka name of the repository where the raw data to the experiment is available",
+      "name": "experiment_typpe",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_context"]
+    },
+    {
+      "description": "String identifier aka name of the repository where the raw data to the experiment is available",
+      "name": "data_repository_name",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": ""
+    },
+    {
+      "description": "URL of this repository",
+      "name": "where",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": ""
+    },
+    {
+      "description": "Thumbnail image informing about the experiment",
+      "name": "data_preview_url",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": ""
+    },
+    {
+      "description": "Shape of the None/Null object",
+      "name": "none_shape",
+      "dtypeStr": "i",
+      "kindStr": "type_dimension",
+      "shape": [],
+      "superNames": ["section_experiment"]
+    },
+    {
+      "description": "Full name of the experimental method in use",
+      "name": "experiment_method",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": ""
+    },
+    {
+      "description": "Name of the city and country the experiment took place, format 'Country, City'",
+      "name": "experiment_location",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": ""
+    },
+    {
+      "description": "Name of the institution hosting the experimental facility",
+      "name": "experiment_facility_institution",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": ""
+    },
+    {
+      "description": "Name of the equipment, instrument with which the experiment was performed e.g. LEAP5000XS",
+      "name": "experiment_tool_info",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": ""
+    },
+    {
+      "description": "UTC start time of the experiment, format 'DD.MM.YYYY - HH.MM.SS'",
+      "name": "experiment_date_global_start",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": "C"
+    },
+    {
+      "description": "UTC end time of the experiment, format 'DD.MM.YYYY - HH.MM.SS'",
+      "name": "experiment_date_global_end",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": "C"
+    },
+    {
+      "description": "Local start time of the experiment, format 'DD.MM.YYYY - HH.MM.SS'",
+      "name": "experiment_date_local_start",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": "C"
+    },
+    {
+      "description": "Operation mode of the instrument (APT, FIM or combination)",
+      "name": "experiment_operation_method",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": ""
+    },
+    {
+      "description": "Pulsing method to enforce a controlled ion evaporation sequence",
+      "name": "experiment_imaging_method",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": ""
+    },
+    {
+      "description": "Sample description e.g. pure W wire samples trial 2",
+      "name": "specimen_description",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": ""
+    },
+    {
+      "description": "Number of elements (disjoint element names) expected",
+      "name": "number_of_elements",
+      "dtypeStr": "i",
+      "kindStr": "type_dimension",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": ""
+    },
+    {
+      "description": "List of periodic table names expected contained in dataset",
+      "name": "specimen_chemistry",
+      "dtypeStr": "C",
+      "shape": ["number_of_elements"],
+      "superNames": ["section_experiment"],
+      "units": ""
+    },
+    {
+      "description": "Qualitative type of specimen and microstructure analyzed (e.g. thin films, nano objects, single crystal, polycrystal)",
+      "name": "specimen_microstructure",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": ""
+    },
+    {
+      "description": "Qualitative information how many phases in the specimen",
+      "name": "specimen_constitution",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": ""
+    },
+	{
+      "description": "Number of ions successfully evaporated",
+      "name": "measured_number_ions_evaporated",
+      "dtypeStr": "i",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": "1"
+    },
+    {
+      "description": "Detector hit positions x and y",
+      "name": "measured_detector_hit_pos",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": "mm, mm"
+    },
+    {
+      "description": "Detector hit multiplicity",
+      "name": "measured_detector_hit_mult",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": "1"
+    },
+    {
+      "description": "Detector number of dead pulses",
+      "name": "measured_detector_dead_pulses",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": "1"
+    },
+    {
+      "description": "Raw ion time of flight",
+      "name": "measured_time_of_flight",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": "ns"
+    },
+    {
+      "description": "Standing voltage",
+      "name": "measured_standing_voltage",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": "V"
+    },
+    {
+      "description": "Pulse voltage",
+      "name": "measured_pulse_voltage",
+      "dtypeStr": "C",
+      "shape": [],
+      "superNames": ["section_experiment"],
+      "units": "V"
+    }
+  ]
+}

tests/aptfim.tester.json

+{
+    "data_repository_name": "zenodo.org",
+    "data_repository_url": "http://zenodo.org/record/1249280",
+    "data_preview_url": "http://zenodo.org/record/1249280/preview.png",
+
+    "experiment_method": "atom probe tomography or field ion microscopy",
+    "experiment_location": "Düsseldorf, Germany",
+    "experiment_facility_institution": "Max-Planck-Institut für Eisenforschung GmbH",
+    "instrument_info": "LEAP5000",
+    
+    "experiment_date_global_start": "01.01.2000 11:11:11",
+    "experiment_date_global_end": "01.01.2000 22:22:22",
+    "experiment_data_local_start": "01.01.2000 11:11:11",
+    "experiment_operation_method": "apt",
+    "experiment_imaging_method": "laser",
+    
+    "specimen_description": "Pure W wire samples bulk polycrystal",
+    "specimen_chemistry": ["Fe", "Mn"],
+    "specimen_microstructure": "single crystal",
+    "specimen_constitution": "single phase",
+    
+    "measured_nions_evaporated": 129000,
+    "measured_detector_hit_pos": "yes",
+    "measured_detector_hit_mult": "no",
+    "measured_detector_dead_pulses": "no",
+    "measured_time_of_flight": "yes",
+    "measured_standing_voltage": "yes",
+    "measured_pulse_voltage": "no"
+}