Added sub-sections to be allowed in required parameters in entry metadata API.

gui/src/components/api.js

@@ -513,7 +513,7 @@ class Api {
     this.onStartLoading()
     return this.swagger()
       .then(client => client.apis.repo.search({
-        exclude: ['atoms', 'only_atoms', 'dft.files', 'dft.quantities', 'dft.optimade', 'dft.labels', 'dft.geometries'],
+        exclude: ['atoms', 'only_atoms', 'files', 'dft.quantities', 'dft.optimade', 'dft.labels', 'dft.geometries'],
         ...search}))
       .catch(handleApiError)
       .then(response => response.body)

nomad/app/v1/models.py

@@ -29,7 +29,7 @@ import fnmatch
 from nomad import datamodel  # pylint: disable=unused-import
 from nomad.utils import strip
 from nomad.metainfo import Datetime, MEnum
-from nomad.metainfo.search_extension import metrics, search_quantities
+from nomad.metainfo.search_extension import metrics, search_quantities, search_sub_sections
 
 from .utils import parameter_dependency_from_model
 
@@ -416,8 +416,8 @@ class MetadataRequired(BaseModel):
             return None
 
         for item in value:
-            assert item in search_quantities or item[-1] == '*', \
-                'required fields must be valid search quantities or contain wildcards'
+            assert item in search_quantities or item in search_sub_sections or item[-1] == '*', \
+                f'required fields ({item}) must be valid search quantities or contain wildcards'
 
         if field.name == 'include' and 'calc_id' not in value:
             value.append('calc_id')

nomad/datamodel/datamodel.py

@@ -571,10 +571,10 @@ class EntryMetadata(metainfo.MSection):
         description='The number of atoms in the entry\'s material',
         a_search=Search())
 
-    ems = metainfo.SubSection(sub_section=EMSMetadata, a_search='ems')
-    dft = metainfo.SubSection(sub_section=DFTMetadata, a_search='dft', categories=[FastAccess])
-    qcms = metainfo.SubSection(sub_section=QCMSMetadata, a_search='qcms')
-    encyclopedia = metainfo.SubSection(sub_section=EncyclopediaMetadata, categories=[FastAccess], a_search='encyclopedia')
+    ems = metainfo.SubSection(sub_section=EMSMetadata, a_search=Search())
+    dft = metainfo.SubSection(sub_section=DFTMetadata, a_search=Search(), categories=[FastAccess])
+    qcms = metainfo.SubSection(sub_section=QCMSMetadata, a_search=Search())
+    encyclopedia = metainfo.SubSection(sub_section=EncyclopediaMetadata, categories=[FastAccess], a_search=Search())
 
     def apply_user_metadata(self, metadata: dict):
         ''' Applies a user provided metadata dict to this calc. '''

nomad/datamodel/dft.py

@@ -240,7 +240,7 @@ class DFTMetadata(MSection):
     labels = SubSection(
         sub_section=Label, repeats=True, categories=[FastAccess],
         description='The labels taken from AFLOW prototypes and springer.',
-        a_search='labels')
+        a_search=Search())
 
     labels_springer_compound_class = Quantity(
         type=str, shape=['0..*'],
@@ -259,7 +259,7 @@ class DFTMetadata(MSection):
     optimade = SubSection(
         sub_section=OptimadeEntry,
         description='Metadata used for the optimade API.',
-        a_search='optimade')
+        a_search=Search())
 
     workflow = Quantity(type=Workflow, a_search=Search())
 

nomad/datamodel/encyclopedia.py

@@ -593,7 +593,7 @@ class Properties(MSection):
         """,
         a_search=Search()
     )
-    energies = SubSection(sub_section=Energies.m_def, repeats=False, categories=[FastAccess], a_search='energies')
+    energies = SubSection(sub_section=Energies.m_def, repeats=False, categories=[FastAccess], a_search=Search())
     electronic_band_structure = Quantity(
         type=Reference(section_k_band.m_def),
         shape=[],
@@ -639,15 +639,15 @@ class Properties(MSection):
 class EncyclopediaMetadata(MSection):
     m_def = Section(
         a_flask=dict(skip_none=True),
-        a_search='encyclopedia',
+        a_search=Search(),
         description="""
         Section which stores information for the NOMAD Encyclopedia.
         """
     )
-    material = SubSection(sub_section=Material.m_def, repeats=False, categories=[FastAccess], a_search='material')
-    method = SubSection(sub_section=Method.m_def, repeats=False, categories=[FastAccess], a_search='method')
-    properties = SubSection(sub_section=Properties.m_def, repeats=False, categories=[FastAccess], a_search='properties')
-    calculation = SubSection(sub_section=Calculation.m_def, repeats=False, categories=[FastAccess], a_search='calculation')
+    material = SubSection(sub_section=Material.m_def, repeats=False, categories=[FastAccess], a_search=Search())
+    method = SubSection(sub_section=Method.m_def, repeats=False, categories=[FastAccess], a_search=Search())
+    properties = SubSection(sub_section=Properties.m_def, repeats=False, categories=[FastAccess], a_search=Search())
+    calculation = SubSection(sub_section=Calculation.m_def, repeats=False, categories=[FastAccess], a_search=Search())
     status = Quantity(
         type=MEnum("success", "unsupported_material_type", "unsupported_method_type", "unsupported_calculation_type", "invalid_metainfo", "failure"),
         description="""

nomad/datamodel/material.py

@@ -137,7 +137,7 @@ class Properties(MSection):
         """,
         a_search=Search()
     )
-    energies = SubSection(sub_section=Energies.m_def, repeats=False, a_search='energies')
+    energies = SubSection(sub_section=Energies.m_def, repeats=False, a_search=Search())
     has_electronic_band_structure = Quantity(
         type=bool,
         shape=[],

nomad/metainfo/elastic_extension.py

@@ -162,14 +162,17 @@ class ElasticDocument(SectionAnnotation):
             inner_document = ElasticDocument.create_document(
                 sub_section.sub_section, prefix=sub_section_prefix, index_name=index_name, root=False)
             if inner_document is not None:
-                try:
-                    if sub_section.a_search.nested:
+                for annotation in sub_section.m_get_annotations(Elastic, as_list=True):
+                    if annotation.nested:
                         assert sub_section.repeats, (
                             "Nested fields should be repeatable. If the subsection cannot be repeated, "
                             "define it as unnested instead."
                         )
                         attrs[sub_section.name] = Nested(inner_document)
-                except AttributeError:
+
+                    annotation.register(prefix, annotation.field, index_name)
+
+                if sub_section.name not in attrs:
                     attrs[sub_section.name] = Object(inner_document)
 
         # create an field for each quantity

nomad/metainfo/search_extension.py

@@ -19,13 +19,16 @@
 from typing import Callable, Any, Dict, List, DefaultDict
 from collections import defaultdict
 
-from nomad import config
+from nomad import config, metainfo
 from nomad.metainfo.elastic_extension import Elastic
 
 
 search_quantities_by_index: DefaultDict[str, Dict[str, 'Search']] = defaultdict(dict)
 ''' All available search quantities by their full qualified name. '''
 
+search_sub_sections_by_index: DefaultDict[str, Dict[str, 'Search']] = defaultdict(dict)
+''' All available sub sections in the search index with full qualified name. '''
+
 metrics_by_index: DefaultDict[str, Dict[str, 'Search']] = defaultdict(dict)
 '''
 The available search metrics. Metrics are integer values given for each entry that can
@@ -43,6 +46,7 @@ order_default_quantities_by_index: DefaultDict[str, Dict[str, 'Search']] = defau
 
 
 search_quantities = search_quantities_by_index[config.elastic.index_name]
+search_sub_sections = search_sub_sections_by_index[config.elastic.index_name]
 groups = groups_by_index[config.elastic.index_name]
 metrics = metrics_by_index[config.elastic.index_name]
 order_default_quantities = order_default_quantities_by_index[config.elastic.index_name]
@@ -154,6 +158,11 @@ class Search(Elastic):
         else:
             self.search_field = self.qualified_name
 
+        if self.definition.m_def == metainfo.SubSection.m_def:
+            if not self.nested:
+                search_sub_sections_by_index[index][self.qualified_name] = self
+                return
+
         assert self.qualified_name not in search_quantities_by_index[index], 'Search quantities must have a unique name: %s' % self.name
         search_quantities_by_index[index][self.qualified_name] = self
 

tests/app/flask/test_api.py

@@ -1176,7 +1176,7 @@ class TestRepo():
             assert value in statistics['dft.system']
 
     def test_search_exclude(self, api, example_elastic_calcs, no_warn):
-        rv = api.get('/repo/?exclude=atoms,only_atoms')
+        rv = api.get('/repo/?exclude=atoms,only_atoms,dft.optimade,dft.quantities')
         assert rv.status_code == 200
         result = search.flat(json.loads(rv.data)['results'][0])
         assert 'atoms' not in result

tests/app/v1/conftest.py

@@ -24,5 +24,4 @@ from nomad.app.main import app
 
 @pytest.fixture(scope='session')
 def client():
-    print('###')
     return TestClient(app, base_url='http://testserver/api/v1/')

tests/app/v1/routers/test_entries.py

@@ -549,6 +549,8 @@ def test_entries_aggregations(client, data, test_user_auth, aggregation, total,
     pytest.param({'exclude': ['upload_id']}, 200, id='exclude'),
     pytest.param({'exclude': ['missspelled', 'upload_id']}, 422, id='bad-quantitiy'),
     pytest.param({'exclude': ['calc_id']}, 200, id='exclude-id'),
+    pytest.param({'exclude': ['dft.optimade']}, 200, id='exclude-sub-section'),
+    pytest.param({'exclude': ['files', 'dft.optimade', 'dft.quantities']}, 200, id='exclude-multiple'),
     pytest.param({'include': ['upload_id']}, 200, id='include-id')
 ])
 @pytest.mark.parametrize('http_method', ['post', 'get'])

tests/metainfo/test_metainfo.py

@@ -286,26 +286,31 @@ class TestM2:
             def new(self, section):
                 return dict(test='test annotation')
 
-        class TestQuantityAnnotation(DefinitionAnnotation):
+        class TestDefinitionAnnotation(DefinitionAnnotation):
             def init_annotation(self, definition):
                 super().init_annotation(definition)
-                assert definition.name in ['test_quantity', 'list_test_quantity']
+                assert definition.name in ['test_quantity', 'list_test_quantity', 'test_sub_section']
                 assert definition.m_parent is not None
                 self.initialized = True
 
         class TestSection(MSection):
             m_def = Section(a_test=TestSectionAnnotation())
 
-            test_quantity = Quantity(type=str, a_test=TestQuantityAnnotation())
+            test_quantity = Quantity(type=str, a_test=TestDefinitionAnnotation())
             list_test_quantity = Quantity(
                 type=str,
-                a_test=[TestQuantityAnnotation(), TestQuantityAnnotation()])
+                a_test=[TestDefinitionAnnotation(), TestDefinitionAnnotation()])
+
+            test_sub_section = SubSection(sub_section=System, a_test=TestDefinitionAnnotation())
 
         assert TestSection.m_def.a_test.initialized
         assert TestSection.m_def.m_get_annotations(TestSectionAnnotation).initialized
-
         assert TestSection().a_test == 'test annotation'
 
+        assert TestSection.test_quantity.a_test is not None
+        assert len(TestSection.list_test_quantity.m_get_annotations(TestDefinitionAnnotation)) == 2
+        assert TestSection.test_sub_section.a_test is not None
+
 
 class TestM1:
     ''' Test for meta-info instances. '''