Commit 56b5839f authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Refactored public/common packages into common_dft.

parent 5e225fbe
Pipeline #87988 passed with stages
in 27 minutes and 53 seconds
......@@ -358,12 +358,8 @@ Consequently, the parser class implementation is modified as in the following ex
```python
import json
from .metainfo import m_env
from nomad.parsing.parser import MatchingParser
from nomad.datamodel.metainfo.common_experimental import section_experiment as msection_experiment
from nomad.datamodel.metainfo.common_experimental import section_data as msection_data
from nomad.datamodel.metainfo.general_experimental_method import section_method as msection_method
from nomad.datamodel.metainfo.general_experimental_sample import section_sample as msection_sample
from nomad.datamodel.metainfo.common_experimental import Experiment, Data, Method, Sample
class ExampleParser(MatchingParser):
......@@ -374,34 +370,32 @@ class ExampleParser(MatchingParser):
)
def run(self, filepath, logger=None):
self._metainfo_env = m_env
with open(filepath, 'rt') as f:
data = json.load(f)
section_experiment = msection_experiment()
experiment = Experiment()
# Read general tool environment details
section_experiment.experiment_location = data.get('experiment_location')
section_experiment.experiment_facility_institution = data.get('experiment_facility_institution')
experiment.experiment_location = data.get('experiment_location')
experiment.experiment_facility_institution = data.get('experiment_facility_institution')
# Read data parameters
section_data = section_experiment.m_create(msection_data)
section_data = experiment.m_create(Data)
section_data.data_repository_name = data.get('data_repository_name')
section_data.data_preview_url = data.get('data_repository_url')
# Read parameters related to method
section_method = section_experiment.m_create(msection_method)
section_method = experiment.m_create(Method)
section_method.experiment_method_name = data.get('experiment_method')
section_method.probing_method = 'electric pulsing'
# Read parameters related to sample
section_sample = section_experiment.m_create(msection_sample)
section_sample = experiment.m_create(Sample)
section_sample.sample_description = data.get('specimen_description')
section_sample.sample_microstructure = data.get('specimen_microstructure')
section_sample.sample_constituents = data.get('specimen_constitution')
return section_experiment
return experiment
```
The parser extends the ``MatchingParser`` class which already implements the determination
of the necessary file for parsing. The main difference to the old framework is the absense
......
......@@ -67,6 +67,10 @@ If you bookmark this page, you can save the definition represented by the highli
To learn more about the meta-info, visit the [meta-info documentation](${appBase}/docs/metainfo.html).
`
function defCompare(a, b) {
return a.name.localeCompare(b.name)
}
export const metainfoConfigState = atom({
key: 'metainfoConfig',
default: {
......@@ -206,7 +210,7 @@ export class PackagePrefixAdaptor extends MetainfoAdaptor {
return <Content>
<Compartment title="Sections">
{sectionDefs.filter(def => !def.extends_base_section).map(def => {
{sectionDefs.filter(def => !def.extends_base_section).sort(defCompare).map(def => {
const key = `section_definitions@${def._qualifiedName}`
return <Item key={key} itemKey={key}>
<Typography>{def.name}</Typography>
......@@ -214,7 +218,7 @@ export class PackagePrefixAdaptor extends MetainfoAdaptor {
})}
</Compartment>
<Compartment title="Section Extensions">
{sectionDefs.filter(def => def.extends_base_section).map(def => {
{sectionDefs.filter(def => def.extends_base_section).sort(defCompare).map(def => {
const key = `section_definitions@${def._qualifiedName}`
return <Item key={key} itemKey={key}>
<Typography>{def.name}</Typography>
......@@ -222,7 +226,7 @@ export class PackagePrefixAdaptor extends MetainfoAdaptor {
})}
</Compartment>
<Compartment title="Categories">
{categoryDefs.map(def => {
{categoryDefs.sort(defCompare).map(def => {
const key = `category_definitions@${def._qualifiedName}`
return <Item key={key} itemKey={key}>
<Typography>{def.name}</Typography>
......@@ -323,8 +327,10 @@ function SectionDef({def}) {
{def.sub_sections.filter(filter)
.map(subSectionDef => {
const key = subSectionDef.name
const categories = subSectionDef.categories?.map(c => resolveRef(c))
const unused = categories?.find(c => c.name === 'Unused')
return <Item key={key} itemKey={key}>
<Typography component="span">
<Typography component="span" color={unused && 'error'}>
<Box fontWeight="bold" component="span">
{subSectionDef.name}
</Box>{subSectionDef.repeats && <span>&nbsp;(repeats)</span>}
......@@ -337,9 +343,11 @@ function SectionDef({def}) {
{def.quantities.filter(filter)
.map(quantityDef => {
const key = quantityDef.name
const categories = quantityDef.categories?.map(c => resolveRef(c))
const unused = categories?.find(c => c.name === 'Unused')
return <Item key={key} itemKey={key}>
<Box component="span" whiteSpace="nowrap">
<Typography component="span">
<Typography component="span" color={unused && 'error'}>
<Box fontWeight="bold" component="span">
{quantityDef.name}
</Box>
......
......@@ -29,7 +29,7 @@ import re
from nomad import utils, config, infrastructure
from nomad.metainfo import Quantity, Reference, Definition, MSection, Section, SubSection
from nomad.datamodel import EntryArchive
from nomad.datamodel.metainfo.public import fast_access
from nomad.datamodel.metainfo.common_dft import FastAccess
'''
The archive storage is made from two tiers. First the whole archive is stored in
......@@ -681,7 +681,7 @@ def filter_archive(
def create_partial_archive(archive: EntryArchive) -> Dict:
'''
Creates a partial archive JSON serializable dict that can be stored directly.
The given archive is filtered based on the metainfo category ``fast_access``.
The given archive is filtered based on the metainfo category ``FastAccess``.
Selected sections and other data that they reference (recursively) comprise the
resulting partial archive.
......@@ -708,10 +708,10 @@ def create_partial_archive(archive: EntryArchive) -> Dict:
if section.m_def == EntryArchive.m_def:
if definition.m_def == Quantity:
return True
return fast_access.m_def in definition.categories
return FastAccess.m_def in definition.categories
if isinstance(definition, Quantity) and isinstance(definition.type, Reference) \
and fast_access.m_def in definition.categories:
and FastAccess.m_def in definition.categories:
# Reference list in partial archives are not supported
if definition.is_scalar:
referenced = getattr(section, definition.name)
......@@ -719,7 +719,7 @@ def create_partial_archive(archive: EntryArchive) -> Dict:
referenceds.append(referenced)
if isinstance(definition, SubSection):
return fast_access.m_def in definition.categories
return FastAccess.m_def in definition.categories
return True
......@@ -897,13 +897,13 @@ def compute_required_with_referenced(required):
prop = key.split('[')[0]
prop_definition = parent.all_properties[prop]
if isinstance(prop_definition, SubSection):
if fast_access.m_def not in prop_definition.categories:
if FastAccess.m_def not in prop_definition.categories:
raise _Incomplete()
traverse(value, prop_definition.sub_section)
if isinstance(prop_definition, Quantity) and isinstance(prop_definition.type, Reference):
current[prop] = '*'
if fast_access.m_def not in prop_definition.categories:
if FastAccess.m_def not in prop_definition.categories:
continue
target_section_def = prop_definition.type.target_section_def.m_resolved()
......
......@@ -73,6 +73,7 @@ def metainfo_undecorated():
import nomad.datamodel.ems
import nomad.datamodel.optimade
import nomad.datamodel.encyclopedia
from nomad.parsing import LegacyParser
nomad.metainfo.metainfo.m_package.__init_metainfo__()
nomad.datamodel.datamodel.m_package.__init_metainfo__()
nomad.datamodel.dft.m_package.__init_metainfo__() # pylint: disable=no-member
......@@ -83,7 +84,8 @@ def metainfo_undecorated():
# Ensure all parser metainfo is loaded
from nomad.parsing.parsers import parsers
for parser in parsers:
_ = parser.metainfo_env
if isinstance(parser, LegacyParser):
_ = parser.metainfo_env
export = Environment()
for package in Package.registry.values():
......@@ -121,57 +123,6 @@ def search_quantities():
print(json.dumps(export, indent=2))
@dev.command(help='Generates source-code for the new metainfo from .json files of the old.')
@click.argument('path', nargs=-1)
def legacy_metainfo(path):
from nomad.metainfo.legacy import convert, generate_metainfo_code
if len(path) == 0:
path = [
'abinit.nomadmetainfo.json',
'aptfim.nomadmetainfo.json',
'atk.nomadmetainfo.json',
'band.nomadmetainfo.json',
'bigdft.nomadmetainfo.json',
'castep.nomadmetainfo.json',
'cp2k.nomadmetainfo.json',
'cpmd.nomadmetainfo.json',
'crystal.nomadmetainfo.json',
'dl_poly.nomadmetainfo.json',
'dmol3.nomadmetainfo.json',
'eels.nomadmetainfo.json',
'elastic.nomadmetainfo.json',
'elk.nomadmetainfo.json',
'exciting.nomadmetainfo.json',
'fhi_aims.nomadmetainfo.json',
'fleur.nomadmetainfo.json',
'gamess.nomadmetainfo.json',
'gaussian.nomadmetainfo.json',
'gpaw.nomadmetainfo.json',
'gulp.nomadmetainfo.json',
'lib_atoms.nomadmetainfo.json',
'molcas.nomadmetainfo.json',
'mpes.nomadmetainfo.json',
'nwchem.nomadmetainfo.json',
'octopus.nomadmetainfo.json',
'onetep.nomadmetainfo.json',
'orca.nomadmetainfo.json',
'phonopy.nomadmetainfo.json',
'photoemission.nomadmetainfo.json',
'qbox.nomadmetainfo.json',
'quantum_espresso.nomadmetainfo.json',
'siesta.nomadmetainfo.json',
'turbomole.nomadmetainfo.json',
'vasp.nomadmetainfo.json',
'wien2k.nomadmetainfo.json',
'dft.nomadmetainfo.json',
'ems.nomadmetainfo.json']
for element in path:
env = convert(element)
generate_metainfo_code(env)
@dev.command(help='Generates a JSON file that compiles all the parser metadata from each parser project.')
def parser_metadata():
import inspect
......
......@@ -27,7 +27,7 @@ from nomad import metainfo, config
from nomad.metainfo.search_extension import Search
from nomad.metainfo.elastic_extension import ElasticDocument
from nomad.metainfo.mongoengine_extension import Mongo, MongoDocument
from nomad.datamodel.metainfo.public import fast_access
from nomad.datamodel.metainfo.common_dft import FastAccess
from .dft import DFTMetadata
from .ems import EMSMetadata
......@@ -38,9 +38,9 @@ from .qcms import QCMSMetadata
m_package = metainfo.Package()
from .encyclopedia import EncyclopediaMetadata # noqa
from .metainfo.public import section_run, Workflow # noqa
from .metainfo.common_dft import Run, Workflow # noqa
from .metainfo.common_experimental import Experiment # noqa
from .metainfo.general_qcms import QuantumCMS # noqa
from .metainfo.common_qcms import QuantumCMS # noqa
def _only_atoms(atoms):
......@@ -538,9 +538,9 @@ class EntryMetadata(metainfo.MSection):
a_search=Search())
ems = metainfo.SubSection(sub_section=EMSMetadata, a_search='ems')
dft = metainfo.SubSection(sub_section=DFTMetadata, a_search='dft', categories=[fast_access])
dft = metainfo.SubSection(sub_section=DFTMetadata, a_search='dft', categories=[FastAccess])
qcms = metainfo.SubSection(sub_section=QCMSMetadata, a_search='qcms')
encyclopedia = metainfo.SubSection(sub_section=EncyclopediaMetadata, categories=[fast_access], a_search='encyclopedia')
encyclopedia = metainfo.SubSection(sub_section=EncyclopediaMetadata, categories=[FastAccess], a_search='encyclopedia')
def apply_user_metadata(self, metadata: dict):
''' Applies a user provided metadata dict to this calc. '''
......@@ -563,11 +563,11 @@ class EntryMetadata(metainfo.MSection):
class EntryArchive(metainfo.MSection):
section_run = metainfo.SubSection(sub_section=section_run, repeats=True)
section_run = metainfo.SubSection(sub_section=Run, repeats=True)
section_experiment = metainfo.SubSection(sub_section=Experiment)
section_quantum_cms = metainfo.SubSection(sub_section=QuantumCMS)
section_workflow = metainfo.SubSection(sub_section=Workflow, categories=[fast_access])
section_metadata = metainfo.SubSection(sub_section=EntryMetadata, categories=[fast_access])
section_workflow = metainfo.SubSection(sub_section=Workflow, categories=[FastAccess])
section_metadata = metainfo.SubSection(sub_section=EntryMetadata, categories=[FastAccess])
processing_logs = metainfo.Quantity(
type=Any, shape=['0..*'],
......
......@@ -27,8 +27,8 @@ from nomad.metainfo import MSection, Section, Quantity, MEnum, SubSection
from nomad.metainfo.search_extension import Search
from .optimade import OptimadeEntry
from .metainfo.public import Workflow, fast_access
from .metainfo.public import section_XC_functionals
from .metainfo.common_dft import Workflow, FastAccess
from .metainfo.common_dft import section_XC_functionals
xc_treatments = {
......@@ -238,7 +238,7 @@ class DFTMetadata(MSection):
a_search=Search(many_or='append', group='groups_grouped', metric_name='groups', metric='cardinality'))
labels = SubSection(
sub_section=Label, repeats=True, categories=[fast_access],
sub_section=Label, repeats=True, categories=[FastAccess],
description='The labels taken from AFLOW prototypes and springer.',
a_search='labels')
......
......@@ -26,7 +26,7 @@ from nomad.metainfo.search_extension import Search
# due to the next imports requireing the m_package already, this would be too late.
m_package = Package()
from .metainfo.public import section_k_band, section_dos, section_thermodynamical_properties, fast_access # noqa
from .metainfo.common_dft import section_k_band, section_dos, section_thermodynamical_properties, FastAccess # noqa
class WyckoffVariables(MSection):
......@@ -82,7 +82,7 @@ class WyckoffSet(MSection):
Chemical element at this Wyckoff position.
"""
)
variables = SubSection(sub_section=WyckoffVariables.m_def, repeats=False, categories=[fast_access])
variables = SubSection(sub_section=WyckoffVariables.m_def, repeats=False, categories=[FastAccess])
class LatticeParameters(MSection):
......@@ -208,8 +208,8 @@ class IdealizedStructure(MSection):
""",
a_search=Search()
)
wyckoff_sets = SubSection(sub_section=WyckoffSet.m_def, repeats=True, categories=[fast_access])
lattice_parameters = SubSection(sub_section=LatticeParameters.m_def, categories=[fast_access])
wyckoff_sets = SubSection(sub_section=WyckoffSet.m_def, repeats=True, categories=[FastAccess])
lattice_parameters = SubSection(sub_section=LatticeParameters.m_def, categories=[FastAccess])
class Bulk(MSection):
......@@ -388,10 +388,10 @@ class Material(MSection):
)
# Bulk-specific properties
bulk = SubSection(sub_section=Bulk.m_def, repeats=False, categories=[fast_access])
bulk = SubSection(sub_section=Bulk.m_def, repeats=False, categories=[FastAccess])
# The idealized structure for this material
idealized_structure = SubSection(sub_section=IdealizedStructure.m_def, repeats=False, categories=[fast_access])
idealized_structure = SubSection(sub_section=IdealizedStructure.m_def, repeats=False, categories=[FastAccess])
class Method(MSection):
......@@ -593,7 +593,7 @@ class Properties(MSection):
""",
a_search=Search()
)
energies = SubSection(sub_section=Energies.m_def, repeats=False, categories=[fast_access], a_search='energies')
energies = SubSection(sub_section=Energies.m_def, repeats=False, categories=[FastAccess], a_search='energies')
electronic_band_structure = Quantity(
type=Reference(section_k_band.m_def),
shape=[],
......@@ -644,10 +644,10 @@ class EncyclopediaMetadata(MSection):
Section which stores information for the NOMAD Encyclopedia.
"""
)
material = SubSection(sub_section=Material.m_def, repeats=False, categories=[fast_access], a_search='material')
method = SubSection(sub_section=Method.m_def, repeats=False, categories=[fast_access], a_search='method')
properties = SubSection(sub_section=Properties.m_def, repeats=False, categories=[fast_access], a_search='properties')
calculation = SubSection(sub_section=Calculation.m_def, repeats=False, categories=[fast_access], a_search='calculation')
material = SubSection(sub_section=Material.m_def, repeats=False, categories=[FastAccess], a_search='material')
method = SubSection(sub_section=Method.m_def, repeats=False, categories=[FastAccess], a_search='method')
properties = SubSection(sub_section=Properties.m_def, repeats=False, categories=[FastAccess], a_search='properties')
calculation = SubSection(sub_section=Calculation.m_def, repeats=False, categories=[FastAccess], a_search='calculation')
status = Quantity(
type=MEnum("success", "unsupported_material_type", "unsupported_method_type", "unsupported_calculation_type", "invalid_metainfo", "failure"),
description="""
......
......@@ -19,11 +19,7 @@
import sys
from nomad.metainfo import Environment
from nomad.metainfo.legacy import LegacyMetainfoEnvironment
import nomad.datamodel.metainfo.common
import nomad.datamodel.metainfo.public
import nomad.datamodel.metainfo.general
import nomad.datamodel.metainfo.common_dft
m_env = LegacyMetainfoEnvironment()
m_env.m_add_sub_section(Environment.packages, sys.modules['nomad.datamodel.metainfo.common'].m_package) # type: ignore
m_env.m_add_sub_section(Environment.packages, sys.modules['nomad.datamodel.metainfo.public'].m_package) # type: ignore
m_env.m_add_sub_section(Environment.packages, sys.modules['nomad.datamodel.metainfo.general'].m_package) # type: ignore
m_env.m_add_sub_section(Environment.packages, sys.modules['nomad.datamodel.metainfo.common_dft'].m_package) # type: ignore
This diff is collapsed.
This diff is collapsed.
......@@ -23,7 +23,7 @@ from nomad.metainfo import (
Datetime, JSON)
m_package = Package(name='common_experimental')
m_package = Package(name='experimental_common')
class Experiment(MSection):
......
This diff is collapsed.
......@@ -16,164 +16,10 @@
# limitations under the License.
#
import numpy as np # pylint: disable=unused-import
import typing # pylint: disable=unused-import
from nomad.metainfo import ( # pylint: disable=unused-import
MSection, MCategory, Category, Package, Quantity, Section, SubSection, SectionProxy,
Reference
)
from nomad.metainfo.legacy import LegacyDefinition
# This is just for backward compatibility of old imports. All definitions are now in common_dft
from nomad.metainfo import Package
m_package = Package(
name='general_nomadmetainfo_json',
description='None',
a_legacy=LegacyDefinition(name='general.nomadmetainfo.json'))
from .common_dft import *
class section_entry_info(MSection):
'''
General information about this entry that is independent from its domain, field, or
used parser
'''
m_def = Section(validate=False, a_legacy=LegacyDefinition(name='section_entry_info'))
entry_upload_time = Quantity(
type=np.dtype(np.int64),
shape=[],
description='''
Upload datetime, given as total number of seconds is the elapsed since the unix
epoch (1 January 1970)
''',
a_legacy=LegacyDefinition(name='entry_upload_time'))
entry_uploader_name = Quantity(
type=str,
shape=[],
description='''
Name of the uploader, given as lastname, firstname.
''',
a_legacy=LegacyDefinition(name='entry_uploader_name'))
entry_uploader_id = Quantity(
type=str,
shape=[],
description='''
The id of the uploader.
''',
a_legacy=LegacyDefinition(name='entry_uploader_id'))
upload_id = Quantity(
type=str,
shape=[],
description='''
Nomad upload id
''',
a_legacy=LegacyDefinition(name='upload_id'))
calc_id = Quantity(
type=str,
shape=[],
description='''
Nomad calc id.
''',
a_legacy=LegacyDefinition(name='calc_id'))
calc_hash = Quantity(
type=str,
shape=[],
description='''
Calculation hash based on raw file contents.
''',
a_legacy=LegacyDefinition(name='calc_hash'))
mainfile = Quantity(
type=str,
shape=[],
description='''
Path to the main file within the upload.
''',
a_legacy=LegacyDefinition(name='mainfile'))
parser_name = Quantity(
type=str,
shape=[],
description='''
Name of the parser used to extract this information.
''',
a_legacy=LegacyDefinition(name='parser_name'))
filepaths = Quantity(
type=str,
shape=['number_of_files'],
description='''
Filepaths of files that belong to this entry, i.e. files in the same directory.
Filepaths are relative to the upload.
''',
a_legacy=LegacyDefinition(name='filepaths'))
number_of_files = Quantity(
type=int,
shape=[],
description='''
Number of files that belong to this entry.
''',
a_legacy=LegacyDefinition(name='number_of_files'))
section_archive_processing_info = SubSection(
sub_section=SectionProxy('section_archive_processing_info'),
repeats=True,
a_legacy=LegacyDefinition(name='section_archive_processing_info'))
class section_archive_processing_info(MSection):
'''
Information about the used archive processing steps and their execution.
'''
m_def = Section(validate=False, a_legacy=LegacyDefinition(name='section_archive_processing_info'))
archive_processor_name = Quantity(
type=str,
shape=[],
description='''
Name of the applied archive processing program.
''',
a_legacy=LegacyDefinition(name='archive_processor_name'))
archive_processor_error = Quantity(
type=str,
shape=[],
description='''
The main error during execution of the archive processing program that failed the
program.
''',
a_legacy=LegacyDefinition(name='archive_processor_error'))
number_of_archive_processor_warnings = Quantity(
type=int,
shape=[],
description='''
Number of warnings during execution of the archive processing program.
''',
a_legacy=LegacyDefinition(name='number_of_archive_processor_warnings'))
archive_processor_warnings = Quantity(
type=str,
shape=['number_of_archive_processor_warnings'],
description='''
Warnings during execution of the archive processing program.
''',
a_legacy=LegacyDefinition(name='archive_processor_warnings'))
archive_processor_status = Quantity(
type=str,
shape=[],
description='''
Status returned by archive processing program.
''',
a_legacy=LegacyDefinition(name='archive_processor_status'))
m_package.__init_metainfo__()
m_package = Package(name='general_nomadmetainfo_json')
This diff is collapsed.
This diff is collapsed.
#
# Copyright The NOMAD Authors.
#
# This file is part of NOMAD. See https://nomad-lab.eu for further info.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of