Commit 6ec1c403 authored by temok-mx's avatar temok-mx
Browse files

Merge branch 'master' of https://gitlab.mpcdf.mpg.de/mkuehbac/parser-aptfim

Updated README.md. Added metadata.yml. The lead branch is now master, inactive branches became tags
parents c9d5750e b9b8f841
......@@ -19,113 +19,82 @@ import re
import numpy as np
from datetime import datetime
from nomadcore.simple_parser import SimpleMatcher
from nomadcore.baseclasses import ParserInterface, AbstractBaseParser
from nomad.parsing.parser import FairdiParser
from nomad.datamodel.metainfo.common_experimental import (
Experiment, Data, Method, Sample, Location, Material)
from nomad.parsing import LocalBackend
class APTFIMParser(FairdiParser):
def __init__(self):
super().__init__(
name='parsers/aptfim', code_name='mpes', code_homepage='https://github.com/mpes-kit/mpes',
domain='ems', mainfile_mime_re=r'(application/json)|(text/.*)', mainfile_name_re=(r'.*.aptfim')
)
class APTFIMParserInterface(ParserInterface):
def get_metainfo_filename(self):
"""
The parser specific metainfo. To include other metadata definitions, use
the 'dependencies' key to refer to other local nomadmetainfo.json files or
to nomadmetainfo.json files that are part of the general nomad-meta-info
submodule (i.e. ``dependencies/nomad-meta-info``).
"""
return os.path.join(os.path.dirname(__file__), 'aptfim.nomadmetainfo.json')
def get_parser_info(self):
""" Basic info about parser used in archive data and logs. """
return {
'name': 'aptfimparser',
'version': '0.1.0'
}
def setup_version(self):
""" Can be used to call :func:`setup_main_parser` differently for different code versions. """
self.setup_main_parser(None)
def setup_main_parser(self, _):
""" Setup the actual parser (behind this interface) """
self.main_parser = APTFIMParser(self.parser_context)
class APTFIMParser(AbstractBaseParser):
def parse(self, filepath):
backend = self.parser_context.super_backend
def parse(self, filepath, archive, logger=None):
with open(filepath, 'rt') as f:
data = json.load(f)
#print(data)
# # You need to open sections before you can add values or sub sections to it.
# # The returned 'gid' can be used to reference a specific section if multiple
# # sections of the same type are opened.
#test_gid = backend.openSection('experiment_cantext')
#backend.addValue('context_headxx', data.get('experiment_type'))
#backend.closeSection('experiment_context', test_gid)
root_gid = backend.openSection('section_experiment')
# # Values do not necessarily have to be read from the parsed file.
# # The backend will check the type of the given value agains the metadata definition.
# backend.addValue('experiment_time', int(datetime.strptime(data.get('date'), '%d.%M.%Y').timestamp()))
#
# # Read data .
# data_gid = backend.openSection('section_context')
#data_gid = backend.openSection('section_experiment')
# addValue
# first argument STRING IDENTIFIER IN OUTPUT JSON KEYWORDS from aptfim.nomadmetainfo.json (ie. the generated parser result JSON)
# second argument STRING IDENTIFIER IN INPUT JSON (ie. the small META DATA FILE TO THE DATASET
#backend.addValue('data_repository_name', data.get('data_repository_name'))
#backend.addValue('data_repository_url', data.get('data_repository_url'))
#backend.addValue('data_preview_url', data.get('data_preview_url'))
# backend.addValue('real_one', data.get('experiment_typpe'))
# backend.closeSection('section_context', data_gid)
experiment = archive.m_create(Experiment)
experiment.raw_metadata = data
# Read general tool environment details
# general_gid = backend.openSection('section_experiment_general_parameters')
backend.addValue('experiment_method', data.get('experiment_method'))
backend.addValue('experiment_location', data.get('experiment_location'))
backend.addValue('experiment_facility_institution', data.get('experiment_facility_institution'))
backend.addValue('experiment_tool_info', data.get('instrument_info')) ###test here the case that input.json keyword is different to output.json
# backend.addValue('experiment_data_global_start', np.array(re.findall(r"[\w']+", data.get('experiment_data_global_start')))) ####
# backend.addValue('experiment_data_global_end', np.array(re.findall(r"[\w']+", data.get('experiment_data_global_end')))) ####
# backend.addValue('experiment_data_local_start', np.array(re.findall(r"[\w']+", data.get('experiment_data_local_start')))) ####
# backend.addValue('experiment_operation_method', data.get('experiment_operation_method'))
# backend.addValue('experiment_imaging_method', data.get('experiment_imaging_method'))
location = experiment.m_create(Location)
location.address = data.get('experiment_location')
location.facility = data.get('experiment_facility_institution')
experiment.experiment_summary = '%s of %s.' % (
data.get('experiment_method').capitalize(), data.get('specimen_description'))
try:
experiment.experiment_time = datetime.strptime(
data.get('experiment_date_global_start'), '%d.%m.%Y %M:%H:%S')
except ValueError:
pass
try:
experiment.experiment_end_time = datetime.strptime(
data.get('experiment_date_global_end'), '%d.%m.%Y %M:%H:%S')
except ValueError:
pass
# Read data parameters
section_data = experiment.m_create(Data)
section_data.repository_name = data.get('data_repository_name')
section_data.entry_repository_url = data.get('data_repository_url')
section_data.repository_url = '/'.join(data.get('data_repository_url').split('/')[0:3])
preview_url = data.get('data_preview_url')
# TODO: This a little hack to correct the preview url and should be removed
# after urls are corrected
preview_url = '%s/files/%s' % tuple(preview_url.rsplit('/', 1))
section_data.preview_url = preview_url
# Read parameters related to method
method = experiment.m_create(Method)
method.data_type = 'image'
method.method_name = data.get('experiment_method')
method.method_abbreviation = 'APT/FIM'
method.probing_method = 'electric pulsing'
method.instrument_description = data.get('instrument_info')
method.measured_number_ions_evaporated = data.get('measured_number_ions_evaporated')
method.measured_detector_hit_pos = data.get('measured_detector_hit_pos') == 'yes'
method.measured_detector_hit_mult = data.get('measured_detector_hit_mult') == 'yes'
method.measured_detector_dead_pulses = data.get('measured_detector_dead_pulses') == 'yes'
method.measured_time_of_flight = data.get('measured_time_of_flight') == 'yes'
method.measured_standing_voltage = data.get('measured_standing_voltage') == 'yes'
method.measured_pulse_voltage = data.get('measured_pulse_voltage') == 'yes'
method.experiment_operation_method = data.get('experiment_operation_method') == 'yes'
method.experiment_imaging_method = data.get('experiment_imaging_method') == 'yes'
# Read parameters related to sample
# backend.addValue('specimen_description', data.get('specimen_description'))
# backend.addValue('specimen_microstructure', data.get('specimen_microstructure'))
# backend.addValue('specimen_constitution', data.get('specimen_constitution'))
#### parse chemical composition
### measured_pulse_voltage for instance should be a conditional read
# backend.addValue('measured_number_ions_evaporated', data.get('measured_number_ions_evaporated'))
# backend.addValue('measured_detector_hit_pos', data.get('measured_detector_hit_pos'))
# backend.addValue('measured_detector_hit_mult', data.get('measured_detector_hit_mult'))
# backend.addValue('measured_detector_dead_pulses', data.get('measured_detector_dead_pulses'))
# backend.addValue('measured_time_of_flight', data.get('measured_time_of_flight'))
# backend.addValue('measured_standing_voltage', data.get('measured_standing_voltage'))
# backend.addValue('measured_pulse_voltage', data.get('measured_pulse_voltage'))
# To add arrays (vectors, matrices, etc.) use addArrayValues and provide a
# numpy array. The shape of the numpy array must match the shape defined in
# the respective metadata definition.
# Close sections in the reverse order
#backend.closeSection('section_experiment', data_gid)
#backend.closeSection('section_data', data_gid)
backend.closeSection('section_experiment', root_gid)
# backend.closeSection('section_experiment_general_parameters', general_gid)
# backend.closeSection('section_experiment_source_parameters', source_gid)
# backend.closeSection('section_experiment_detector_parameters', detector_gid)
# backend.closeSection('section_experiment_sample_parameters', sample_gid)
sample = experiment.m_create(Sample)
sample.sample_description = data.get('specimen_description')
sample.sample_microstructure = data.get('specimen_microstructure')
sample.sample_constituents = data.get('specimen_constitution')
material = sample.m_create(Material)
atom_labels = data.get('specimen_chemistry')
formula = ase.Atoms(atom_labels).get_chemical_formula()
material.atom_labels = np.array(atom_labels)
material.chemical_formula = formula
material.chemical_name = formula
......@@ -12,15 +12,10 @@
# limitations under the License.
import sys
import json
from nomad.parsing import LocalBackend
from aptfimparser import APTFIMParserInterface
from aptfimparser import APTFIMParser
if __name__ == "__main__":
# instantiate the parser via its interface with a LocalBackend
parser = APTFIMParserInterface(backend=LocalBackend)
# call the actual parsing with the given mainfile
parser.parse(sys.argv[1])
# print the results stored in the LocalBackend
parser.parser_context.super_backend.write_json(
sys.stdout, pretty=True, root_sections=['section_experiment'])
archive = APTFIMParser.main(sys.argv[1])
json.dump(archive.m_to_dict(), sys.stdout, indent=2)
{
"type": "nomad_meta_info_1_0",
"description": "Metadata for an atom probe tomography or field ion microscopy experiment.",
"dependencies":[
{
"metainfoPath":"general.nomadmetainfo.json"
},
{
"metainfoPath":"general.experimental.nomadmetainfo.json"
}
],
"metaInfos": [
{
"description": "String identifier aka name of the repository where the raw data to the experiment is available",
"name": "data_repository_name",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": ""
},
{
"description": "URL of this repository",
"name": "data_repository_url",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": ""
},
{ "description": "Thumbnail image informing about the experiment",
"name": "data_preview_url",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": ""
},
{
"description": "Shape of the None/Null object",
"name": "none_shape",
"dtypeStr": "i",
"kindStr": "type_dimension",
"shape": [],
"superNames": ["section_experiment"]
},
{
"description": "Full name of the experimental method in use",
"name": "experiment_method",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": ""
},
{
"description": "Name of the city and country the experiment took place, format 'Country, City'",
"name": "experiment_location",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": ""
},
{
"description": "Name of the institution hosting the experimental facility",
"name": "experiment_facility_institution",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": ""
},
{
"description": "Name of the equipment, instrument with which the experiment was performed e.g. LEAP5000XS",
"name": "experiment_tool_info",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": ""
},
{
"description": "UTC start time of the experiment, format 'DD.MM.YYYY - HH.MM.SS'",
"name": "experiment_date_global_start",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": "C"
},
{
"description": "UTC end time of the experiment, format 'DD.MM.YYYY - HH.MM.SS'",
"name": "experiment_date_global_end",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": "C"
},
{
"description": "Local start time of the experiment, format 'DD.MM.YYYY - HH.MM.SS'",
"name": "experiment_date_local_start",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": "C"
},
{
"description": "Operation mode of the instrument (APT, FIM or combination)",
"name": "experiment_operation_method",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": ""
},
{
"description": "Pulsing method to enforce a controlled ion evaporation sequence",
"name": "experiment_imaging_method",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": ""
},
{
"description": "Sample description e.g. pure W wire samples trial 2",
"name": "specimen_description",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": ""
},
{
"description": "Number of elements (disjoint element names) expected",
"name": "number_of_elements",
"dtypeStr": "i",
"kindStr": "type_dimension",
"shape": [],
"superNames": ["section_experiment"],
"units": ""
},
{
"description": "List of periodic table names expected contained in dataset",
"name": "specimen_chemistry",
"dtypeStr": "C",
"shape": ["number_of_elements"],
"superNames": ["section_experiment"],
"units": ""
},
{
"description": "Qualitative type of specimen and microstructure analyzed (e.g. thin films, nano objects, single crystal, polycrystal)",
"name": "specimen_microstructure",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": ""
},
{
"description": "Qualitative information how many phases in the specimen",
"name": "specimen_constitution",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": ""
},
{
"description": "Number of ions successfully evaporated",
"name": "measured_number_ions_evaporated",
"dtypeStr": "i",
"shape": [],
"superNames": ["section_experiment"],
"units": "1"
},
{
"description": "Detector hit positions x and y",
"name": "measured_detector_hit_pos",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": "mm, mm"
},
{
"description": "Detector hit multiplicity",
"name": "measured_detector_hit_mult",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": "1"
},
{
"description": "Detector number of dead pulses",
"name": "measured_detector_dead_pulses",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": "1"
},
{
"description": "Raw ion time of flight",
"name": "measured_time_of_flight",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": "ns"
},
{
"description": "Standing voltage",
"name": "measured_standing_voltage",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": "V"
},
{
"description": "Pulse voltage",
"name": "measured_pulse_voltage",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": "V"
}
]
}
import sys
from nomad.metainfo import Environment
from nomad.metainfo.legacy import LegacyMetainfoEnvironment
import aptfimparser.metainfo.aptfim
import nomad.datamodel.metainfo.general
import nomad.datamodel.metainfo.common_experimental
m_env = LegacyMetainfoEnvironment()
m_env.m_add_sub_section(Environment.packages, sys.modules['aptfimparser.metainfo.aptfim'].m_package) # type: ignore
m_env.m_add_sub_section(Environment.packages, sys.modules['nomad.datamodel.metainfo.general'].m_package) # type: ignore
m_env.m_add_sub_section(Environment.packages, sys.modules['nomad.datamodel.metainfo.common_experimental'].m_package) # type: ignore
from nomad.metainfo import Package, Quantity, Section
from nomad.datamodel.metainfo import common_experimental
m_package = Package(name='aptfim')
class Method(common_experimental.Method):
m_def = Section(validate=False, extends_base_section=True)
experiment_operation_method = Quantity(
type=str,
description='Operation mode of the instrument (APT, FIM or combination)')
experiment_imaging_method = Quantity(
type=str,
description='Pulsing method to enforce a controlled ion evaporation sequence')
number_ions_evaporated = Quantity(
type=int,
description='Number of ions successfully evaporated')
measured_detector_hit_pos = Quantity(
type=bool,
description='Detector hit positions x and y was measured')
measured_detector_hit_mult = Quantity(
type=bool,
description='Detector hit multiplicity was measured')
measured_detector_dead_pulses = Quantity(
type=bool,
description='Detector number of dead pulses was measured')
measured_time_of_flight = Quantity(
type=bool,
description='Raw ion time of flight was measured')
measured_standing_voltage = Quantity(
type=bool,
description='Standing voltage was measured')
measured_pulse_voltage = Quantity(
type=bool,
description='Pulse voltage was measured')
m_package.__init_metainfo__()
{
"type": "nomad_meta_info_1_0",
"description": "Metadata for an atom probe tomography or field ion microscopy experiment.",
"dependencies":[
{
"metainfoPath":"general.nomadmetainfo.json"
},
{
"metainfoPath":"general.experimental.nomadmetainfo.json"
}
],
"metaInfos": [
{
"description": "String identifier aka name of the repository where the raw data to the experiment is available",
"name": "experiment_typpe",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_context"]
},
{
"description": "String identifier aka name of the repository where the raw data to the experiment is available",
"name": "data_repository_name",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": ""
},
{
"description": "URL of this repository",
"name": "where",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": ""
},
{
"description": "Thumbnail image informing about the experiment",
"name": "data_preview_url",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": ""
},
{
"description": "Shape of the None/Null object",
"name": "none_shape",
"dtypeStr": "i",
"kindStr": "type_dimension",
"shape": [],
"superNames": ["section_experiment"]
},
{
"description": "Full name of the experimental method in use",
"name": "experiment_method",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": ""
},
{
"description": "Name of the city and country the experiment took place, format 'Country, City'",
"name": "experiment_location",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": ""
},
{
"description": "Name of the institution hosting the experimental facility",
"name": "experiment_facility_institution",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": ""
},
{
"description": "Name of the equipment, instrument with which the experiment was performed e.g. LEAP5000XS",
"name": "experiment_tool_info",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": ""
},
{
"description": "UTC start time of the experiment, format 'DD.MM.YYYY - HH.MM.SS'",
"name": "experiment_date_global_start",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": "C"
},
{
"description": "UTC end time of the experiment, format 'DD.MM.YYYY - HH.MM.SS'",
"name": "experiment_date_global_end",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": "C"
},
{
"description": "Local start time of the experiment, format 'DD.MM.YYYY - HH.MM.SS'",
"name": "experiment_date_local_start",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": "C"
},
{
"description": "Operation mode of the instrument (APT, FIM or combination)",
"name": "experiment_operation_method",
"dtypeStr": "C",
"shape": [],
"superNames": ["section_experiment"],
"units": ""
},
{
"description": "Pulsing method to enforce a controlled ion evaporation sequence",
"name": "experiment_imaging_method",
"dtypeStr": "C",