Commit 13ff696f authored by Aviral Vaid's avatar Aviral Vaid Committed by Markus Scheidgen
Browse files

Modified common_experimental.py, ems.py

parent 19bcd811
Subproject commit 58d6cfa16d14093a1c00d23ec097a3f370c9756a
Subproject commit 49417ec2a747eb4467f69e7f41e24d7d5fae66dc
......@@ -40,7 +40,7 @@ m_package = metainfo.Package()
from .encyclopedia import EncyclopediaMetadata # noqa
from .metainfo.common_dft import Run, Workflow # noqa
from .metainfo.common_experimental import Experiment # noqa
from .metainfo.common_experimental import Measurement # noqa
from .metainfo.common_qcms import QuantumCMS # noqa
......@@ -602,7 +602,8 @@ class EntryMetadata(metainfo.MSection):
class EntryArchive(metainfo.MSection):
section_run = metainfo.SubSection(sub_section=Run, repeats=True)
section_experiment = metainfo.SubSection(sub_section=Experiment)
# section_experiment = metainfo.SubSection(sub_section=Experiment)
section_measurement = metainfo.SubSection(sub_section=Measurement, repeats=True)
section_quantum_cms = metainfo.SubSection(sub_section=QuantumCMS)
section_workflow = metainfo.SubSection(sub_section=Workflow, categories=[FastAccess])
section_metadata = metainfo.SubSection(sub_section=EntryMetadata, categories=[FastAccess])
......@@ -611,7 +612,6 @@ class EntryArchive(metainfo.MSection):
type=Any, shape=['0..*'],
description='The processing logs for this entry as a list of structlog entries.')
# preemptively create the elasticsearch document definition, which populates metrics and
# search quantities in the search_extension
EntryMetadata.m_def.a_elastic.document
......@@ -67,9 +67,9 @@ class EMSMetadata(MSection):
entry = self.m_parent
root_section = entry_archive.section_experiment
entry.formula = root_section.section_sample.section_material.chemical_formula
atoms = root_section.section_sample.section_material.atom_labels
root_section = entry_archive.section_measurement[0]
entry.formula = root_section.section_metadata.section_sample.formula
atoms = root_section.section_metadata.section_sample.elements
if atoms is None:
entry.atoms = []
......@@ -82,34 +82,34 @@ class EMSMetadata(MSection):
atoms.sort()
entry.atoms = atoms
self.chemical = _unavailable(root_section.section_sample.section_material.chemical_name)
self.sample_microstructure = _unavailable(root_section.section_sample.sample_microstructure)
self.sample_constituents = _unavailable(root_section.section_sample.sample_constituents)
self.experiment_summary = root_section.experiment_summary
location = root_section.experiment_location
if location is not None:
location_str = ', '.join([
getattr(location, prop)
for prop in ['facility', 'institution', 'address']
if getattr(location, prop) is not None])
self.experiment_location = location_str
if root_section.experiment_time:
self.origin_time = root_section.experiment_time
elif root_section.experiment_publish_time:
self.origin_time = root_section.experiment_publish_time
# self.chemical = _unavailable(root_section.section_sample.section_material.chemical_name)
# self.sample_microstructure = _unavailable(root_section.section_sample.sample_microstructure)
# self.sample_constituents = _unavailable(root_section.section_sample.sample_constituents)
self.experiment_summary = root_section.section_metadata.section_experiment.notes
# location = root_section.experiment_location
# if location is not None:
# location_str = ', '.join([
# getattr(location, prop)
# for prop in ['facility', 'institution', 'address']
# if getattr(location, prop) is not None])
# self.experiment_location = location_str
if root_section.section_metadata.section_experiment.experiment_start_time:
self.origin_time = root_section.section_metadata.section_experiment.experiment_start_time
elif root_section.section_metadata.section_experiment.experiment_publish_time:
self.origin_time = root_section.section_metadata.section_experiment.experiment_publish_time
else:
self.origin_time = self.m_parent.upload_time
self.data_type = _unavailable(root_section.section_method.data_type)
self.method = _unavailable(root_section.section_method.method_name)
self.probing_method = _unavailable(root_section.section_method.probing_method)
# self.data_type = _unavailable(root_section.section_method.data_type)
self.method = _unavailable(root_section.section_metadata.section_experiment.method_name)
# self.probing_method = _unavailable(root_section.section_method.probing_method)
self.repository_name = _unavailable(root_section.section_data.repository_name)
self.repository_url = root_section.section_data.repository_url
self.preview_url = root_section.section_data.preview_url
self.entry_repository_url = root_section.section_data.entry_repository_url
self.repository_name = _unavailable(root_section.section_metadata.section_origin.repository_name)
self.repository_url = _unavailable(root_section.section_metadata.section_origin.repository_url)
self.preview_url = _unavailable(root_section.section_metadata.section_origin.preview_url)
self.entry_repository_url = _unavailable(root_section.section_metadata.section_origin.entry_repository_url)
self.group_hash = utils.hash(
entry.formula,
......
......@@ -19,69 +19,92 @@
import numpy as np
from nomad.metainfo import (
MSection, Package, Quantity, SubSection, SectionProxy, Section,
MSection, MCategory, Package, Quantity, SubSection,
Datetime, JSON)
m_package = Package(name='experimental_common')
class Experiment(MSection):
'''
The root section for all (meta)data that belongs to a single experiment.
'''
m_def = Section(validate=False)
class UserProvided(MCategory):
pass
class DeviceSettings(MSection):
device_name = Quantity(type=str)
analysis_method = Quantity(type=str)
analyzer_lens = Quantity(type=str)
analyzer_slit = Quantity(type=str)
scan_mode = Quantity(type=str)
detector_voltage = Quantity(type=str)
workfunction = Quantity(type=str)
channel_id = Quantity(type=str)
max_energy = Quantity(type=str)
min_energy = Quantity(type=str)
guntype = Quantity(type=str)
beam_energy = Quantity(type=str)
resolution = Quantity(type=str)
step_size = Quantity(type=str)
acquisition_mode = Quantity(type=str)
beam_current = Quantity(type=str)
detector_type = Quantity(type=str)
dark_current = Quantity(type=str)
notes = Quantity(type=str, categories=[UserProvided])
experiment_summary = Quantity(
type=str, description='A descriptive summary of the content of the experiment.')
class Sample(MSection):
spectrum_region = Quantity(type=str, shape=[])
sample_id = Quantity(type=str)
formula = Quantity(type=str)
elements = Quantity(type=str, shape=['*'])
sample_title = Quantity(type=str)
notes = Quantity(type=str, categories=[UserProvided])
experiment_location = SubSection(sub_section=SectionProxy('Location'))
class Experiment(MSection):
method_name = Quantity(type=str)
method_abbreviation = Quantity(type=str)
experiment_id = Quantity(type=str)
experiment_publish_time = Quantity(
type=Datetime, description='The datetime when this experiment was published.')
experiment_time = Quantity(
experiment_start_time = Quantity(
type=Datetime, description='The datetime of the beginning of the experiment.')
experiment_end_time = Quantity(
type=Datetime, description='The datetime of the experiment end.')
edges = Quantity(type=str, shape=['*'])
description = Quantity(type=str)
notes = Quantity(type=str, categories=[UserProvided])
raw_metadata = Quantity(
type=JSON, description='The whole or partial metadata in its original source JSON format.')
class Instrument(MSection):
n_scans = Quantity(type=str)
dwell_time = Quantity(type=str)
excitation_energy = Quantity(type=str)
source_label = Quantity(type=str)
notes = Quantity(type=str, categories=[UserProvided])
section_data = SubSection(sub_section=SectionProxy('Data'))
section_device_settings = SubSection(sub_section=DeviceSettings, repeats=True)
section_method = SubSection(sub_section=SectionProxy('Method'))
section_sample = SubSection(sub_section=SectionProxy('Sample'))
class Author(MSection):
author_name = Quantity(type=str)
author_profile_url = Quantity(type=str)
author_profile_api_url = Quantity(type=str)
group_name = Quantity(type=str)
notes = Quantity(type=str, categories=[UserProvided])
class Location(MSection):
m_def = Section(validate=False)
class DataHeader(MSection):
channel_id = Quantity(type=int)
label = Quantity(type=str)
unit = Quantity(type=str)
notes = Quantity(type=str, categories=[UserProvided])
address = Quantity(
type=str, description='''
The address where the experiment took place, format 'Country, City, Street'
''')
institution = Quantity(
type=str, description='''
Name of the institution hosting the experimental facility (e.g. in full or an
acronym).
''')
facility = Quantity(
type=str, description='''
Name of the experimental facility (e.g. in full or an acronym).
''')
class Data(MSection):
'''
This section contains information about the stored data.
'''
m_def = Section(validate=False)
class Origin(MSection):
section_author = SubSection(sub_section=Author, repeats=True)
permalink = Quantity(type=str)
api_permalink = Quantity(type=str)
repository_name = Quantity(
type=str, description='The name of the repository, where the data is stored.')
......@@ -93,79 +116,181 @@ class Data(MSection):
entry_repository_url = Quantity(
type=str, description='An URL to the entry on the repository, where the data is stored.')
notes = Quantity(type=str, categories=[UserProvided])
class Method(MSection):
'''
This section contains information about the applied experimental method.
'''
m_def = Section(validate=False)
class Spectrum(MSection):
n_values = Quantity(type=int)
excitation_energy_expected = Quantity(type=np.dtype(np.float64), shape=['n_values'], unit='J', description='The expected excitation energy range of the spectrum')
excitation_energy_actual = Quantity(type=np.dtype(np.float64), shape=['n_values'], unit='J', description='The actual excitation energy range of the spectrum')
count = Quantity(type=np.dtype(np.float64), shape=['n_values'], description='The count at each energy value, dimensionless')
ring_current = Quantity(type=np.dtype(np.float64), shape=['n_values'], unit='A', description='Ring current')
total_electron_yield = Quantity(type=np.dtype(np.float64), shape=['n_values'], unit='V', description='Total electron yield')
mirror_current = Quantity(type=np.dtype(np.float64), shape=['n_values'], unit='V', description='Mirror current')
energy = Quantity(type=np.dtype(np.float64), shape=['n_values'], unit='J', description='The energy range of the (EELS) spectrum')
notes = Quantity(type=str, categories=[UserProvided])
data_type = Quantity(
type=str, description='Name of the type of data that the experiment produced.')
method_name = Quantity(
type=str, description='Full name of the experimental method in use')
class Metadata(MSection):
section_sample = SubSection(sub_section=Sample)
section_experiment = SubSection(sub_section=Experiment)
section_instrument = SubSection(sub_section=Instrument)
section_data_header = SubSection(sub_section=DataHeader)
section_origin = SubSection(sub_section=Origin)
notes = Quantity(type=str, categories=[UserProvided])
method_abbreviation = Quantity(
type=str,
description='Abbreviated name (i.e. acronym) of the experimental method')
probing_method = Quantity(
type=str, description='The probing method used')
class Data(MSection):
section_spectrum = SubSection(sub_section=Spectrum)
notes = Quantity(type=str, categories=[UserProvided])
instrument_description = Quantity(
type=str, description='A description of the instrumentation used for the experiment.')
class Measurement(MSection):
section_metadata = SubSection(sub_section=Metadata)
section_data = SubSection(sub_section=Data)
description = Quantity(type=str, categories=[UserProvided])
class Sample(MSection):
'''
The section for all sample related (meta)data that was used in the experiment.
'''
m_def = Section(validate=False)
sample_description = Quantity(
type=str, description='Description of the sample used in the experiment.')
# class Experiment(MSection):
# '''
# The root section for all (meta)data that belongs to a single experiment.
# '''
# m_def = Section(validate=False)
# experiment_summary = Quantity(
# type=str, description='A descriptive summary of the content of the experiment.')
# experiment_location = SubSection(sub_section=SectionProxy('Location'))
# experiment_publish_time = Quantity(
# type=Datetime, description='The datetime when this experiment was published.')
# experiment_time = Quantity(
# type=Datetime, description='The datetime of the beginning of the experiment.')
# experiment_end_time = Quantity(
# type=Datetime, description='The datetime of the experiment end.')
# raw_metadata = Quantity(
# type=JSON, description='The whole or partial metadata in its original source JSON format.')
# section_data = SubSection(sub_section=SectionProxy('Data'))
# section_method = SubSection(sub_section=SectionProxy('Method'))
# section_sample = SubSection(sub_section=SectionProxy('Sample'))
# class Location(MSection):
# m_def = Section(validate=False)
# address = Quantity(
# type=str, description='''
# The address where the experiment took place, format 'Country, City, Street'
# ''')
# institution = Quantity(
# type=str, description='''
# Name of the institution hosting the experimental facility (e.g. in full or an
# acronym).
# ''')
# facility = Quantity(
# type=str, description='''
# Name of the experimental facility (e.g. in full or an acronym).
# ''')
# class Data(MSection):
# '''
# This section contains information about the stored data.
# '''
# m_def = Section(validate=False)
# repository_name = Quantity(
# type=str, description='The name of the repository, where the data is stored.')
# repository_url = Quantity(
# type=str, description='An URL to the repository, where the data is stored.')
# preview_url = Quantity(
# type=str, description='An URL to an image file that contains a preview.')
# entry_repository_url = Quantity(
# type=str, description='An URL to the entry on the repository, where the data is stored.')
# class Method(MSection):
# '''
# This section contains information about the applied experimental method.
# '''
# m_def = Section(validate=False)
# data_type = Quantity(
# type=str, description='Name of the type of data that the experiment produced.')
# method_name = Quantity(
# type=str, description='Full name of the experimental method in use')
# method_abbreviation = Quantity(
# type=str,
# description='Abbreviated name (i.e. acronym) of the experimental method')
# probing_method = Quantity(
# type=str, description='The probing method used')
# instrument_description = Quantity(
# type=str, description='A description of the instrumentation used for the experiment.')
# class Sample(MSection):
# '''
# The section for all sample related (meta)data that was used in the experiment.
# '''
# m_def = Section(validate=False)
# sample_description = Quantity(
# type=str, description='Description of the sample used in the experiment.')
sample_id = Quantity(
type=str, description='Identification number or signatures of the sample used.')
# sample_id = Quantity(
# type=str, description='Identification number or signatures of the sample used.')
sample_state = Quantity(
type=str, description='The physical state of the sample.')
# sample_state = Quantity(
# type=str, description='The physical state of the sample.')
sample_temperature = Quantity(
type=np.dtype(np.float64), unit='kelvin',
description='The temperature of the sample during the experiment.')
# sample_temperature = Quantity(
# type=np.dtype(np.float64), unit='kelvin',
# description='The temperature of the sample during the experiment.')
sample_microstructure = Quantity(
type=str, description='The sample microstructure.')
# sample_microstructure = Quantity(
# type=str, description='The sample microstructure.')
sample_constituents = Quantity(
type=str, description='The constituents.')
# sample_constituents = Quantity(
# type=str, description='The constituents.')
section_material = SubSection(sub_section=SectionProxy('Material'))
# section_material = SubSection(sub_section=SectionProxy('Material'))
class Material(MSection):
''' This section describes a sample's material. '''
m_def = Section(validate=False)
# class Material(MSection):
# ''' This section describes a sample's material. '''
# m_def = Section(validate=False)
chemical_formula = Quantity(
type=str, description='The chemical formula that describes the sample.')
# chemical_formula = Quantity(
# type=str, description='The chemical formula that describes the sample.')
chemical_name = Quantity(
type=str, description='The chemical name that describes the sample.')
# chemical_name = Quantity(
# type=str, description='The chemical name that describes the sample.')
atom_labels = Quantity(
type=str, shape=['number_of_elements'],
description='Atom labels for distinct elements in the sample.')
# atom_labels = Quantity(
# type=str, shape=['number_of_elements'],
# description='Atom labels for distinct elements in the sample.')
number_of_elements = Quantity(
type=int, derived=lambda m: len(m.atom_labels) if m.atom_labels else 0,
description='Number of distinct chemical elements in the sample.')
# number_of_elements = Quantity(
# type=int, derived=lambda m: len(m.atom_labels) if m.atom_labels else 0,
# description='Number of distinct chemical elements in the sample.')
space_group = Quantity(
type=int, description='Space group of the sample compound (if crystalline).')
# space_group = Quantity(
# type=int, description='Space group of the sample compound (if crystalline).')
m_package.__init_metainfo__()
......@@ -25,8 +25,9 @@ from .legacy import LegacyParser
from .artificial import EmptyParser, GenerateRandomParser, TemplateParser, ChaosParser
from eelsdbconverter import EELSApiJsonConverter
from mpesparser import MPESParser
from aptfimparser import APTFIMParser
# TODO
# from mpesparser import MPESParser
# from aptfimparser import APTFIMParser
from vaspparser import VASPParser
from phonopyparser import PhonopyParser
from elasticparser import ElasticParser
......@@ -43,6 +44,7 @@ from octopusparser import OctopusParser
from orcaparser import OrcaParser
from cp2kparser import CP2KParser
from fhivibesparser import FHIVibesParser
from xpsparser import XPSParser
try:
# these packages are not available without parsing extra, which is ok, if the
......@@ -285,9 +287,10 @@ parsers = [
mainfile_contents_re=(
r'Copyright \(C\) [0-9]+ TURBOMOLE GmbH, Karlsruhe')
),
MPESParser(),
APTFIMParser(),
# MPESParser(),
# APTFIMParser(),
EELSApiJsonConverter(),
XPSParser(),
LegacyParser(
name='parsers/qbox', code_name='qbox', code_homepage='http://qboxcode.org/', domain='dft',
parser_class_name='qboxparser.QboxParser',
......
......@@ -566,9 +566,9 @@ def test_ems_data(proc_infra, test_user):
upload = run_processing(('test_ems_upload', 'tests/data/proc/examples_ems.zip'), test_user)
additional_keys = [
'ems.method', 'ems.data_type', 'formula', 'n_atoms', 'atoms', 'ems.chemical', 'ems.origin_time']
assert upload.total_calcs == 3
assert len(upload.calcs) == 3
'ems.method', 'formula', 'n_atoms', 'atoms', 'ems.origin_time']
assert upload.total_calcs == 1
assert len(upload.calcs) == 1
with upload.entries_metadata() as entries:
assert_upload_files(upload.upload_id, entries, StagingUploadFiles, published=False)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment