Commit 03f1c8f2 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added sub-sections to be allowed in required parameters in entry metadata API.

parent 414fdcea
Pipeline #91241 passed with stages
in 27 minutes and 20 seconds
......@@ -513,7 +513,7 @@ class Api {
this.onStartLoading()
return this.swagger()
.then(client => client.apis.repo.search({
exclude: ['atoms', 'only_atoms', 'dft.files', 'dft.quantities', 'dft.optimade', 'dft.labels', 'dft.geometries'],
exclude: ['atoms', 'only_atoms', 'files', 'dft.quantities', 'dft.optimade', 'dft.labels', 'dft.geometries'],
...search}))
.catch(handleApiError)
.then(response => response.body)
......
......@@ -29,7 +29,7 @@ import fnmatch
from nomad import datamodel # pylint: disable=unused-import
from nomad.utils import strip
from nomad.metainfo import Datetime, MEnum
from nomad.metainfo.search_extension import metrics, search_quantities
from nomad.metainfo.search_extension import metrics, search_quantities, search_sub_sections
from .utils import parameter_dependency_from_model
......@@ -416,8 +416,8 @@ class MetadataRequired(BaseModel):
return None
for item in value:
assert item in search_quantities or item[-1] == '*', \
'required fields must be valid search quantities or contain wildcards'
assert item in search_quantities or item in search_sub_sections or item[-1] == '*', \
f'required fields ({item}) must be valid search quantities or contain wildcards'
if field.name == 'include' and 'calc_id' not in value:
value.append('calc_id')
......
......@@ -571,10 +571,10 @@ class EntryMetadata(metainfo.MSection):
description='The number of atoms in the entry\'s material',
a_search=Search())
ems = metainfo.SubSection(sub_section=EMSMetadata, a_search='ems')
dft = metainfo.SubSection(sub_section=DFTMetadata, a_search='dft', categories=[FastAccess])
qcms = metainfo.SubSection(sub_section=QCMSMetadata, a_search='qcms')
encyclopedia = metainfo.SubSection(sub_section=EncyclopediaMetadata, categories=[FastAccess], a_search='encyclopedia')
ems = metainfo.SubSection(sub_section=EMSMetadata, a_search=Search())
dft = metainfo.SubSection(sub_section=DFTMetadata, a_search=Search(), categories=[FastAccess])
qcms = metainfo.SubSection(sub_section=QCMSMetadata, a_search=Search())
encyclopedia = metainfo.SubSection(sub_section=EncyclopediaMetadata, categories=[FastAccess], a_search=Search())
def apply_user_metadata(self, metadata: dict):
''' Applies a user provided metadata dict to this calc. '''
......
......@@ -240,7 +240,7 @@ class DFTMetadata(MSection):
labels = SubSection(
sub_section=Label, repeats=True, categories=[FastAccess],
description='The labels taken from AFLOW prototypes and springer.',
a_search='labels')
a_search=Search())
labels_springer_compound_class = Quantity(
type=str, shape=['0..*'],
......@@ -259,7 +259,7 @@ class DFTMetadata(MSection):
optimade = SubSection(
sub_section=OptimadeEntry,
description='Metadata used for the optimade API.',
a_search='optimade')
a_search=Search())
workflow = Quantity(type=Workflow, a_search=Search())
......
......@@ -593,7 +593,7 @@ class Properties(MSection):
""",
a_search=Search()
)
energies = SubSection(sub_section=Energies.m_def, repeats=False, categories=[FastAccess], a_search='energies')
energies = SubSection(sub_section=Energies.m_def, repeats=False, categories=[FastAccess], a_search=Search())
electronic_band_structure = Quantity(
type=Reference(section_k_band.m_def),
shape=[],
......@@ -639,15 +639,15 @@ class Properties(MSection):
class EncyclopediaMetadata(MSection):
m_def = Section(
a_flask=dict(skip_none=True),
a_search='encyclopedia',
a_search=Search(),
description="""
Section which stores information for the NOMAD Encyclopedia.
"""
)
material = SubSection(sub_section=Material.m_def, repeats=False, categories=[FastAccess], a_search='material')
method = SubSection(sub_section=Method.m_def, repeats=False, categories=[FastAccess], a_search='method')
properties = SubSection(sub_section=Properties.m_def, repeats=False, categories=[FastAccess], a_search='properties')
calculation = SubSection(sub_section=Calculation.m_def, repeats=False, categories=[FastAccess], a_search='calculation')
material = SubSection(sub_section=Material.m_def, repeats=False, categories=[FastAccess], a_search=Search())
method = SubSection(sub_section=Method.m_def, repeats=False, categories=[FastAccess], a_search=Search())
properties = SubSection(sub_section=Properties.m_def, repeats=False, categories=[FastAccess], a_search=Search())
calculation = SubSection(sub_section=Calculation.m_def, repeats=False, categories=[FastAccess], a_search=Search())
status = Quantity(
type=MEnum("success", "unsupported_material_type", "unsupported_method_type", "unsupported_calculation_type", "invalid_metainfo", "failure"),
description="""
......
......@@ -137,7 +137,7 @@ class Properties(MSection):
""",
a_search=Search()
)
energies = SubSection(sub_section=Energies.m_def, repeats=False, a_search='energies')
energies = SubSection(sub_section=Energies.m_def, repeats=False, a_search=Search())
has_electronic_band_structure = Quantity(
type=bool,
shape=[],
......
......@@ -162,14 +162,17 @@ class ElasticDocument(SectionAnnotation):
inner_document = ElasticDocument.create_document(
sub_section.sub_section, prefix=sub_section_prefix, index_name=index_name, root=False)
if inner_document is not None:
try:
if sub_section.a_search.nested:
for annotation in sub_section.m_get_annotations(Elastic, as_list=True):
if annotation.nested:
assert sub_section.repeats, (
"Nested fields should be repeatable. If the subsection cannot be repeated, "
"define it as unnested instead."
)
attrs[sub_section.name] = Nested(inner_document)
except AttributeError:
annotation.register(prefix, annotation.field, index_name)
if sub_section.name not in attrs:
attrs[sub_section.name] = Object(inner_document)
# create an field for each quantity
......
......@@ -19,13 +19,16 @@
from typing import Callable, Any, Dict, List, DefaultDict
from collections import defaultdict
from nomad import config
from nomad import config, metainfo
from nomad.metainfo.elastic_extension import Elastic
search_quantities_by_index: DefaultDict[str, Dict[str, 'Search']] = defaultdict(dict)
''' All available search quantities by their full qualified name. '''
search_sub_sections_by_index: DefaultDict[str, Dict[str, 'Search']] = defaultdict(dict)
''' All available sub sections in the search index with full qualified name. '''
metrics_by_index: DefaultDict[str, Dict[str, 'Search']] = defaultdict(dict)
'''
The available search metrics. Metrics are integer values given for each entry that can
......@@ -43,6 +46,7 @@ order_default_quantities_by_index: DefaultDict[str, Dict[str, 'Search']] = defau
search_quantities = search_quantities_by_index[config.elastic.index_name]
search_sub_sections = search_sub_sections_by_index[config.elastic.index_name]
groups = groups_by_index[config.elastic.index_name]
metrics = metrics_by_index[config.elastic.index_name]
order_default_quantities = order_default_quantities_by_index[config.elastic.index_name]
......@@ -154,6 +158,11 @@ class Search(Elastic):
else:
self.search_field = self.qualified_name
if self.definition.m_def == metainfo.SubSection.m_def:
if not self.nested:
search_sub_sections_by_index[index][self.qualified_name] = self
return
assert self.qualified_name not in search_quantities_by_index[index], 'Search quantities must have a unique name: %s' % self.name
search_quantities_by_index[index][self.qualified_name] = self
......
......@@ -1176,7 +1176,7 @@ class TestRepo():
assert value in statistics['dft.system']
def test_search_exclude(self, api, example_elastic_calcs, no_warn):
rv = api.get('/repo/?exclude=atoms,only_atoms')
rv = api.get('/repo/?exclude=atoms,only_atoms,dft.optimade,dft.quantities')
assert rv.status_code == 200
result = search.flat(json.loads(rv.data)['results'][0])
assert 'atoms' not in result
......
......@@ -24,5 +24,4 @@ from nomad.app.main import app
@pytest.fixture(scope='session')
def client():
print('###')
return TestClient(app, base_url='http://testserver/api/v1/')
......@@ -549,6 +549,8 @@ def test_entries_aggregations(client, data, test_user_auth, aggregation, total,
pytest.param({'exclude': ['upload_id']}, 200, id='exclude'),
pytest.param({'exclude': ['missspelled', 'upload_id']}, 422, id='bad-quantitiy'),
pytest.param({'exclude': ['calc_id']}, 200, id='exclude-id'),
pytest.param({'exclude': ['dft.optimade']}, 200, id='exclude-sub-section'),
pytest.param({'exclude': ['files', 'dft.optimade', 'dft.quantities']}, 200, id='exclude-multiple'),
pytest.param({'include': ['upload_id']}, 200, id='include-id')
])
@pytest.mark.parametrize('http_method', ['post', 'get'])
......
......@@ -286,26 +286,31 @@ class TestM2:
def new(self, section):
return dict(test='test annotation')
class TestQuantityAnnotation(DefinitionAnnotation):
class TestDefinitionAnnotation(DefinitionAnnotation):
def init_annotation(self, definition):
super().init_annotation(definition)
assert definition.name in ['test_quantity', 'list_test_quantity']
assert definition.name in ['test_quantity', 'list_test_quantity', 'test_sub_section']
assert definition.m_parent is not None
self.initialized = True
class TestSection(MSection):
m_def = Section(a_test=TestSectionAnnotation())
test_quantity = Quantity(type=str, a_test=TestQuantityAnnotation())
test_quantity = Quantity(type=str, a_test=TestDefinitionAnnotation())
list_test_quantity = Quantity(
type=str,
a_test=[TestQuantityAnnotation(), TestQuantityAnnotation()])
a_test=[TestDefinitionAnnotation(), TestDefinitionAnnotation()])
test_sub_section = SubSection(sub_section=System, a_test=TestDefinitionAnnotation())
assert TestSection.m_def.a_test.initialized
assert TestSection.m_def.m_get_annotations(TestSectionAnnotation).initialized
assert TestSection().a_test == 'test annotation'
assert TestSection.test_quantity.a_test is not None
assert len(TestSection.list_test_quantity.m_get_annotations(TestDefinitionAnnotation)) == 2
assert TestSection.test_sub_section.a_test is not None
class TestM1:
''' Test for meta-info instances. '''
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment