Commit 707a2ecc authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Made workflow metadata searchable.

parent ab645f58
Subproject commit 8c2e56b0d9d17b777b5fa6a61c8e2877444ab603
Subproject commit 31e56cb7b84f9a375d58c96057a2a94f9af1a27c
......@@ -69,68 +69,68 @@
"description": null,
"many": false
},
"labels.label": {
"name": "labels.label",
"dft.labels.label": {
"name": "dft.labels.label",
"description": "The label as a string",
"many": false
},
"labels.type": {
"name": "labels.type",
"dft.labels.type": {
"name": "dft.labels.type",
"description": "The type of the label",
"many": false
},
"labels.source": {
"name": "labels.source",
"dft.labels.source": {
"name": "dft.labels.source",
"description": "The source that this label was taken from.",
"many": false
},
"optimade.elements": {
"name": "optimade.elements",
"dft.optimade.elements": {
"name": "dft.optimade.elements",
"description": "Names of the different elements present in the structure.",
"many": false
},
"optimade.nelements": {
"name": "optimade.nelements",
"dft.optimade.nelements": {
"name": "dft.optimade.nelements",
"description": "Number of different elements in the structure as an integer.",
"many": false
},
"optimade.elements_ratios": {
"name": "optimade.elements_ratios",
"dft.optimade.elements_ratios": {
"name": "dft.optimade.elements_ratios",
"description": "Relative proportions of different elements in the structure.",
"many": false
},
"optimade.chemical_formula_descriptive": {
"name": "optimade.chemical_formula_descriptive",
"dft.optimade.chemical_formula_descriptive": {
"name": "dft.optimade.chemical_formula_descriptive",
"description": "The chemical formula for a structure as a string in a form chosen by the API\nimplementation.",
"many": false
},
"optimade.chemical_formula_reduced": {
"name": "optimade.chemical_formula_reduced",
"dft.optimade.chemical_formula_reduced": {
"name": "dft.optimade.chemical_formula_reduced",
"description": "The reduced chemical formula for a structure as a string with element symbols and\ninteger chemical proportion numbers. The proportion number MUST be omitted if it is 1.",
"many": false
},
"optimade.chemical_formula_hill": {
"name": "optimade.chemical_formula_hill",
"dft.optimade.chemical_formula_hill": {
"name": "dft.optimade.chemical_formula_hill",
"description": "The chemical formula for a structure in Hill form with element symbols followed by\ninteger chemical proportion numbers. The proportion number MUST be omitted if it is 1.",
"many": false
},
"optimade.chemical_formula_anonymous": {
"name": "optimade.chemical_formula_anonymous",
"dft.optimade.chemical_formula_anonymous": {
"name": "dft.optimade.chemical_formula_anonymous",
"description": "The anonymous formula is the chemical_formula_reduced, but where the elements are\ninstead first ordered by their chemical proportion number, and then, in order left to\nright, replaced by anonymous symbols A, B, C, ..., Z, Aa, Ba, ..., Za, Ab, Bb, ... and\nso on.",
"many": false
},
"optimade.dimension_types": {
"name": "optimade.dimension_types",
"dft.optimade.dimension_types": {
"name": "dft.optimade.dimension_types",
"description": "List of three integers. For each of the three directions indicated by the three lattice\nvectors (see property lattice_vectors). This list indicates if the direction is\nperiodic (value 1) or non-periodic (value 0). Note: the elements in this list each\nrefer to the direction of the corresponding entry in lattice_vectors and not\nthe Cartesian x, y, z directions.",
"many": false
},
"optimade.nsites": {
"name": "optimade.nsites",
"dft.optimade.nsites": {
"name": "dft.optimade.nsites",
"description": "An integer specifying the length of the cartesian_site_positions property.",
"many": false
},
"optimade.structure_features": {
"name": "optimade.structure_features",
"dft.optimade.structure_features": {
"name": "dft.optimade.structure_features",
"description": "A list of strings that flag which special features are used by the structure.\n\n- disorder: This flag MUST be present if any one entry in the species list has a\nchemical_symbols list that is longer than 1 element.\n- unknown_positions: This flag MUST be present if at least one component of the\ncartesian_site_positions list of lists has value null.\n- assemblies: This flag MUST be present if the assemblies list is present.",
"many": false
},
......@@ -233,56 +233,7 @@
"dft.code_name": {
"name": "dft.code_name",
"description": "The name of the used code.",
"many": false,
"statistic_size": 46,
"statistic_values": [
"ABINIT",
"Amber",
"ASAP",
"ATK",
"BAND",
"BigDFT",
"CASTEP",
"Charmm",
"CP2K",
"CPMD",
"Crystal",
"DFTb plus",
"DL_POLY",
"DMol3",
"elastic",
"elk",
"exciting",
"FHI-aims",
"fleur",
"fplo",
"GAMESS",
"Gaussian",
"GPAW",
"Gromacs",
"Gromos",
"gulp",
"lammps",
"libAtoms",
"MOLCAS",
"MOPAC",
"Namd",
"NWChem",
"Octopus",
"ONETEP",
"OpenKIM",
"ORCA",
"Phonopy",
"qbox",
"Quantum Espresso",
"Siesta",
"TINKER",
"turbomole",
"VASP",
"WIEN2k",
"unavailable",
"not processed"
]
"many": false
},
"dft.code_version": {
"name": "dft.code_version",
......@@ -342,6 +293,236 @@
"many": true,
"statistic_size": 20
},
"dft.workflow.workflow_type": {
"name": "dft.workflow.workflow_type",
"description": "The type of calculation workflow. Can be one of relaxation, elastic, phonon,\nmolecular dynamics.",
"many": false
},
"dft.workflow.relaxation_energy_tolerance": {
"name": "dft.workflow.relaxation_energy_tolerance",
"description": "The tolerance value in the energy between relaxation steps for convergence.",
"many": false
},
"dft.workflow": {
"name": "dft.workflow",
"description": null,
"many": false
},
"encyclopedia.material.bulk.bravais_lattice": {
"name": "encyclopedia.material.bulk.bravais_lattice",
"description": "The Bravais lattice type in the Pearson notation, where the first\nlowercase letter indicates the crystal system, and the second uppercase\nletter indicates the lattice type. The value can only be one of the 14\ndifferent Bravais lattices in three dimensions.\n\nCrystal system letters:\n\na = Triclinic\nm = Monoclinic\no = Orthorhombic\nt = Tetragonal\nh = Hexagonal and Trigonal\nc = Cubic\n\nLattice type letters:\n\nP = Primitive\nS (A, B, C) = One side/face centred\nI = Body centered\nR = Rhombohedral centring\nF = All faces centred",
"many": false
},
"encyclopedia.material.bulk.crystal_system": {
"name": "encyclopedia.material.bulk.crystal_system",
"description": "The detected crystal system. One of seven possibilities in three dimensions.",
"many": false
},
"encyclopedia.material.bulk.has_free_wyckoff_parameters": {
"name": "encyclopedia.material.bulk.has_free_wyckoff_parameters",
"description": "Whether the material has any Wyckoff sites with free parameters. If a\nmaterials has free Wyckoff parameters, at least some of the atoms are\nnot bound to a particular location in the structure but are allowed to\nmove with possible restrictions set by the symmetry.",
"many": false
},
"encyclopedia.material.bulk.point_group": {
"name": "encyclopedia.material.bulk.point_group",
"description": "Point group in Hermann-Mauguin notation, part of crystal structure\nclassification. There are 32 point groups in three dimensional space.",
"many": false
},
"encyclopedia.material.bulk.space_group_number": {
"name": "encyclopedia.material.bulk.space_group_number",
"description": "Integer representation of the space group, part of crystal structure\nclassification, part of material definition.",
"many": false
},
"encyclopedia.material.bulk.space_group_international_short_symbol": {
"name": "encyclopedia.material.bulk.space_group_international_short_symbol",
"description": "International short symbol notation of the space group.",
"many": false
},
"encyclopedia.material.bulk.structure_prototype": {
"name": "encyclopedia.material.bulk.structure_prototype",
"description": "The prototypical material for this crystal structure.",
"many": false
},
"encyclopedia.material.bulk.structure_type": {
"name": "encyclopedia.material.bulk.structure_type",
"description": "Classification according to known structure type, considering the point\ngroup of the crystal and the occupations with different atom types.",
"many": false
},
"encyclopedia.material.bulk.strukturbericht_designation": {
"name": "encyclopedia.material.bulk.strukturbericht_designation",
"description": "Classification of the material according to the historically grown \"strukturbericht\".",
"many": false
},
"encyclopedia.material.idealized_structure.lattice_parameters.a": {
"name": "encyclopedia.material.idealized_structure.lattice_parameters.a",
"description": "Length of the first basis vector.",
"many": false
},
"encyclopedia.material.idealized_structure.lattice_parameters.b": {
"name": "encyclopedia.material.idealized_structure.lattice_parameters.b",
"description": "Length of the second basis vector.",
"many": false
},
"encyclopedia.material.idealized_structure.lattice_parameters.c": {
"name": "encyclopedia.material.idealized_structure.lattice_parameters.c",
"description": "Length of the third basis vector.",
"many": false
},
"encyclopedia.material.idealized_structure.lattice_parameters.alpha": {
"name": "encyclopedia.material.idealized_structure.lattice_parameters.alpha",
"description": "Angle between second and third basis vector.",
"many": false
},
"encyclopedia.material.idealized_structure.lattice_parameters.beta": {
"name": "encyclopedia.material.idealized_structure.lattice_parameters.beta",
"description": "Angle between first and third basis vector.",
"many": false
},
"encyclopedia.material.idealized_structure.lattice_parameters.gamma": {
"name": "encyclopedia.material.idealized_structure.lattice_parameters.gamma",
"description": "Angle between first and second basis vector.",
"many": false
},
"encyclopedia.material.idealized_structure.cell_volume": {
"name": "encyclopedia.material.idealized_structure.cell_volume",
"description": "Volume of the idealized cell. The cell volume can only be reported\nconsistently after idealization and may not perfectly correspond to the\noriginal simulation cell.",
"many": false
},
"encyclopedia.material.material_type": {
"name": "encyclopedia.material.material_type",
"description": "\"Broad structural classification for the material, e.g. bulk, 2D, 1D... \",",
"many": false
},
"encyclopedia.material.material_id": {
"name": "encyclopedia.material.material_id",
"description": "Search for a particular material by its id.",
"many": false
},
"encyclopedia.material.material_name": {
"name": "encyclopedia.material.material_name",
"description": "Most meaningful name for a material.",
"many": false
},
"encyclopedia.material.formula": {
"name": "encyclopedia.material.formula",
"description": "Formula giving the composition and occurrences of the elements in the\nHill notation. For periodic materials the formula is calculated fom the\nprimitive unit cell.",
"many": false
},
"encyclopedia.material.formula_reduced": {
"name": "encyclopedia.material.formula_reduced",
"description": "Formula giving the composition and occurrences of the elements in the\nHill notation where the number of occurences have been divided by the\ngreatest common divisor.",
"many": false
},
"encyclopedia.material.species_and_counts": {
"name": "encyclopedia.material.species_and_counts",
"description": "The formula separated into individual terms containing both the atom\ntype and count. Used for searching parts of a formula.",
"many": false
},
"encyclopedia.material.species": {
"name": "encyclopedia.material.species",
"description": "The formula separated into individual terms containing only unique atom\nspecies. Used for searching materials containing specific elements.",
"many": false
},
"encyclopedia.method.method_type": {
"name": "encyclopedia.method.method_type",
"description": "Generic name for the used methodology.",
"many": false
},
"encyclopedia.method.core_electron_treatment": {
"name": "encyclopedia.method.core_electron_treatment",
"description": "How the core electrons are described.",
"many": false
},
"encyclopedia.method.functional_long_name": {
"name": "encyclopedia.method.functional_long_name",
"description": "Full identified for the used exchange-correlation functional.",
"many": false
},
"encyclopedia.method.functional_type": {
"name": "encyclopedia.method.functional_type",
"description": "Basic type of the used exchange-correlation functional.",
"many": false
},
"encyclopedia.method.group_eos_id": {
"name": "encyclopedia.method.group_eos_id",
"description": "A fixed length, unique identifier for equation-of-state calculations.\nOnly calculations within the same upload and with a method hash\navailable will be grouped under the same hash.",
"many": false
},
"encyclopedia.method.group_parametervariation_id": {
"name": "encyclopedia.method.group_parametervariation_id",
"description": "A fixed length, unique identifier for calculations where structure is\nidentical but the used computational parameters are varied. Only\ncalculations within the same upload and with a method hash available\nwill be grouped under the same hash.",
"many": false
},
"encyclopedia.properties.energies.energy_total": {
"name": "encyclopedia.properties.energies.energy_total",
"description": "Total energy.",
"many": false
},
"encyclopedia.properties.energies.energy_total_T0": {
"name": "encyclopedia.properties.energies.energy_total_T0",
"description": "Total energy projected to T=0.",
"many": false
},
"encyclopedia.properties.energies.energy_free": {
"name": "encyclopedia.properties.energies.energy_free",
"description": "Free energy.",
"many": false
},
"encyclopedia.properties.atomic_density": {
"name": "encyclopedia.properties.atomic_density",
"description": "Atomic density of the material (atoms/volume).\"",
"many": false
},
"encyclopedia.properties.mass_density": {
"name": "encyclopedia.properties.mass_density",
"description": "Mass density of the material.",
"many": false
},
"encyclopedia.properties.band_gap": {
"name": "encyclopedia.properties.band_gap",
"description": "Band gap value. If multiple spin channels are present, this value is\ntaken from the channel with smallest band gap value.",
"many": false
},
"encyclopedia.properties.band_gap_direct": {
"name": "encyclopedia.properties.band_gap_direct",
"description": "Whether band gap is direct or not. If multiple spin channels are\npresent, this value is taken from the channel with smallest band gap\nvalue.",
"many": false
},
"encyclopedia.properties.electronic_band_structure": {
"name": "encyclopedia.properties.electronic_band_structure",
"description": "Reference to an electronic band structure.",
"many": false
},
"encyclopedia.properties.electronic_dos": {
"name": "encyclopedia.properties.electronic_dos",
"description": "Reference to an electronic density of states.",
"many": false
},
"encyclopedia.properties.phonon_band_structure": {
"name": "encyclopedia.properties.phonon_band_structure",
"description": "Reference to a phonon band structure.",
"many": false
},
"encyclopedia.properties.phonon_dos": {
"name": "encyclopedia.properties.phonon_dos",
"description": "Reference to a phonon density of states.",
"many": false
},
"encyclopedia.properties.thermodynamical_properties": {
"name": "encyclopedia.properties.thermodynamical_properties",
"description": "Reference to a section containing thermodynamical properties.",
"many": false
},
"encyclopedia.calculation.calculation_type": {
"name": "encyclopedia.calculation.calculation_type",
"description": "Defines the type of calculation that was detected for this entry.",
"many": false
},
"encyclopedia.status": {
"name": "encyclopedia.status",
"description": "The final Encyclopedia processing status for this entry. The meaning of the status is as follows:\n\n| Status | Description |\n| -------------------------------- | ----------------------------------------------------------------------------- |\n| `\"success\"` | Processed successfully |\n| `\"unsupported_material_type\"` | The detected material type is currently not supported by the Encyclopedia. |\n| `\"unsupported_calculation_type\"` | The detected calculation type is currently not supported by the Encyclopedia. |\n| `\"invalid_metainfo\"` | The entry could not be processed due to missing or invalid metainfo. |\n| `\"failure\"` | The entry could not be processed due to an unexpected exception. |",
"many": false
},
"upload_id": {
"name": "upload_id",
"description": "A random UUID that uniquely identifies the upload of the entry.",
......
......@@ -79,11 +79,21 @@ section. We currently have the following domain specific metadata classes/sectio
.. automodule:: nomad.datamodel.encyclopedia
:members:
'''
import sys
from nomad.metainfo import Environment
from .dft import DFTMetadata
from .ems import EMSMetadata
from .datamodel import Dataset, User, EditableUserMetadata, MongoMetadata, EntryMetadata, EntryArchive
from .optimade import OptimadeEntry, Species
from .metainfo import m_env
m_env.m_add_sub_section(Environment.packages, sys.modules['nomad.datamodel.datamodel'].m_package) # type: ignore
m_env.m_add_sub_section(Environment.packages, sys.modules['nomad.datamodel.dft'].m_package) # type: ignore
m_env.m_add_sub_section(Environment.packages, sys.modules['nomad.datamodel.ems'].m_package) # type: ignore
m_env.m_add_sub_section(Environment.packages, sys.modules['nomad.datamodel.encyclopedia'].m_package) # type: ignore
m_env.m_add_sub_section(Environment.packages, sys.modules['nomad.datamodel.optimade'].m_package) # type: ignore
domains = {
'dft': {
......@@ -98,4 +108,4 @@ domains = {
}
}
root_sections = [domain['root_section'] for domain in domains.values()] + ['section_entry_info', 'OptimadeEntry']
root_sections = [domain['root_section'] for domain in domains.values()] + ['section_entry_info', 'OptimadeEntry', 'Workflow']
......@@ -32,7 +32,7 @@ from .ems import EMSMetadata
m_package = metainfo.Package()
from .encyclopedia import EncyclopediaMetadata # noqa
from .metainfo.public import section_run # noqa
from .metainfo.public import section_run, Workflow # noqa
from .metainfo.general_experimental import section_experiment # noqa
......@@ -480,6 +480,7 @@ class EntryArchive(metainfo.MSection):
section_run = metainfo.SubSection(sub_section=section_run, repeats=True)
section_experiment = metainfo.SubSection(sub_section=section_experiment)
section_workflow = metainfo.SubSection(sub_section=Workflow)
section_metadata = metainfo.SubSection(sub_section=EntryMetadata)
processing_logs = metainfo.Quantity(
......
......@@ -24,6 +24,7 @@ from nomad.metainfo.search_extension import Search
from .common import get_optional_backend_value
from .optimade import OptimadeEntry
from .metainfo.public import Workflow
xc_treatments = {
......@@ -263,6 +264,8 @@ class DFTMetadata(MSection):
description='Metadata used for the optimade API.',
a_search='optimade')
workflow = Quantity(type=Workflow, a_search=Search())
def code_name_from_parser(self):
entry = self.m_parent
if entry.parser_name is not None:
......@@ -404,3 +407,6 @@ class DFTMetadata(MSection):
if aflow_id is not None and aflow_label is not None:
self.labels.append(Label(label=aflow_label, type='prototype', source='aflow_prototype_library'))
self.labels.append(Label(label=aflow_id, type='prototype_id', source='aflow_prototype_library'))
if backend.entry_archive.section_workflow:
self.workflow = backend.entry_archive.section_workflow
......@@ -9,9 +9,3 @@ m_env = LegacyMetainfoEnvironment()
m_env.m_add_sub_section(Environment.packages, sys.modules['nomad.datamodel.metainfo.common'].m_package) # type: ignore
m_env.m_add_sub_section(Environment.packages, sys.modules['nomad.datamodel.metainfo.public'].m_package) # type: ignore
m_env.m_add_sub_section(Environment.packages, sys.modules['nomad.datamodel.metainfo.general'].m_package) # type: ignore
m_env.m_add_sub_section(Environment.packages, sys.modules['nomad.datamodel.datamodel'].m_package) # type: ignore
m_env.m_add_sub_section(Environment.packages, sys.modules['nomad.datamodel.dft'].m_package) # type: ignore
m_env.m_add_sub_section(Environment.packages, sys.modules['nomad.datamodel.ems'].m_package) # type: ignore
m_env.m_add_sub_section(Environment.packages, sys.modules['nomad.datamodel.encyclopedia'].m_package) # type: ignore
m_env.m_add_sub_section(Environment.packages, sys.modules['nomad.datamodel.optimade'].m_package) # type: ignore
......@@ -4,6 +4,7 @@ from nomad.metainfo import ( # pylint: disable=unused-import
MSection, MCategory, Category, Package, Quantity, Section, SubSection, SectionProxy,
Reference, MEnum, derived
)
from nomad.metainfo.search_extension import Search
from nomad.metainfo.legacy import LegacyDefinition
......@@ -3439,11 +3440,6 @@ class section_run(MSection):
repeats=True,
a_legacy=LegacyDefinition(name='section_system'))
section_workflow = SubSection(
sub_section=SectionProxy('section_workflow'),
repeats=True
)
class section_sampling_method(MSection):
'''
......@@ -5580,7 +5576,7 @@ class section_XC_functionals(MSection):
a_legacy=LegacyDefinition(name='XC_functional_weight'))
class section_workflow(MSection):
class Workflow(MSection):
'''
Section containing the results of a workflow.
'''
......@@ -5594,7 +5590,8 @@ class section_workflow(MSection):
The type of calculation workflow. Can be one of relaxation, elastic, phonon,
molecular dynamics.
''',
a_legacy=LegacyDefinition(name='workflow_type'))
a_legacy=LegacyDefinition(name='workflow_type'),
a_search=Search())
relaxation_energy_tolerance = Quantity(
type=np.dtype(np.float64),
......@@ -5603,14 +5600,16 @@ class section_workflow(MSection):
description='''
The tolerance value in the energy between relaxation steps for convergence.
''',
a_legacy=LegacyDefinition(name='relaxation_energy_tolerance'))
a_legacy=LegacyDefinition(name='relaxation_energy_tolerance'),
a_search=Search())
workflow_final_calculation_ref = Quantity(
type=Reference(SectionProxy('section_single_configuration_calculation')),
shape=[],
description='''
Reference to last calculation step.
''')
''',
a_legacy=LegacyDefinition(name='workflow_final_calculation_ref'))
m_package.__init_metainfo__()
......@@ -183,9 +183,12 @@ class ElasticDocument(SectionAnnotation):
elif quantity.type == Datetime:
annotation.mapping = Date(**kwargs)
elif isinstance(quantity.type, Reference):
inner_prefix = annotation.field
if prefix is not None:
inner_prefix = '%s.%s' % (prefix, inner_prefix)
inner_document = ElasticDocument.create_document(
cast(Section, quantity.type.target_section_def), inner_doc=True,
prefix=annotation.field)
prefix=inner_prefix)
annotation.mapping = Object(inner_document)
elif isinstance(quantity.type, MEnum):
annotation.mapping = Keyword(**kwargs)
......
......@@ -120,8 +120,9 @@ class TemplateParser(ArtificalParser):
self.backend.pwarn('A test warning.')
template_json = json.load(open(mainfile, 'r'))
section = template_json['section_run'][0]
self.add_section(section)
self.add_section(template_json['section_run'][0])
if 'section_workflow' in template_json:
self.add_section(template_json['section_workflow'])
self.backend.finishedParsingSession('ParseSuccess', [])
logger.debug('a test log entry')
return self.backend
......
......@@ -49,6 +49,10 @@ from nomad.datamodel.encyclopedia import (
import phonopyparser.metainfo
section_metadata = datamodel.EntryArchive.section_metadata.name
section_workflow = datamodel.EntryArchive.section_workflow.name
def _pack_log_event(logger, method_name, event_dict):
try:
log_data = dict(event_dict)
......@@ -176,8 +180,15 @@ class Calc(Proc):
'''
archive = upload_files.read_archive(self.calc_id)
try:
entry_metadata = datamodel.EntryMetadata.m_from_dict(
archive[self.calc_id][datamodel.EntryArchive.section_metadata.name].to_dict())
# instead of loading the whole archive, it should be enough to load the
# parts that are referenced by section_metadata/EntryMetadata
# TODO somehow it should determine which root setions too load from the metainfo
# or configuration
calc_archive = archive[self.calc_id]
entry_archive_dict = {section_metadata: calc_archive[section_metadata].to_dict()}
if section_workflow in calc_archive:
entry_archive_dict[section_workflow] = calc_archive[section_workflow].to_dict()
entry_metadata = datamodel.EntryArchive.m_from_dict(entry_archive_dict)[section_metadata]
except KeyError:
# Due hard processing failures, it might be possible that an entry might not
......
{
"section_workflow": {
"_name": "Workflow",
"_gIndex": 0,
"workflow_type": "geometry_optimization",
"relaxation_energy_tolerance": 0.413754846285984e-24,
"workflow_final_calculation_ref": 0
},