Added metadata from parsing backend to upload_files.

nomad/parsing/backend.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import TextIO, Tuple, List, Any, Callable
+from typing import TextIO, Tuple, List, Any, Callable, Iterable
 from abc import ABCMeta, abstractmethod
 from io import StringIO
 import json
@@ -489,6 +489,27 @@ class LocalBackend(LegacyParserBackend):
         else:
             json_writer.value(value)
 
+    def _obj(self, value: Any, filter: Callable[[str, Any], Any] = None) -> Any:
+        if isinstance(value, list):
+            if len(value) == 1 and isinstance(value[0], Section) and not self._delegate.metaInfoEnv().infoKindEl(value[0].name).repeats:
+                return self._obj(value[0])
+            else:
+                return [self._obj(item) for item in value]
+
+        elif isinstance(value, Section):
+            section = value
+            obj = dict(_name=section.name, _gIndex=section.gIndex)
+            for name, value in section.items():
+                if filter is not None:
+                    value = filter(name, value)
+
+                if value is not None:
+                    obj[name] = self._obj(value, filter=filter)
+            return obj
+
+        else:
+            return JSONStreamWriter._json_serializable_value(value)
+
     @property
     def status(self) -> ParserStatus:
         """ Returns status and potential errors. """
@@ -499,6 +520,13 @@ class LocalBackend(LegacyParserBackend):
         self._errors = None
         self._warnings: List[str] = []
 
+    def metadata(self, sections: Iterable[str] = ['section_repository_info']) -> dict:
+        """ Returns an json serializable object with all data of the given sections. """
+        data = dict(calc_id=self.get_value('calc_id'))
+        for section in sections:
+            data[section] = self._obj(self._delegate.results[section])
+        return data
+
     def write_json(self, out: TextIO, pretty=True, filter: Callable[[str, Any], Any] = None):
         """
         Writes the results stored in the backend after parsing in an 'archive'.json
@@ -516,8 +544,8 @@ class LocalBackend(LegacyParserBackend):
         for root_section in ['section_run', 'section_calculation_info', 'section_repository_info']:
             json_writer.key(root_section)
             json_writer.open_array()
-            for run in self._delegate.results[root_section]:
-                LocalBackend._write(json_writer, run, filter=filter)
+            for section in self._delegate.results[root_section]:
+                LocalBackend._write(json_writer, section, filter=filter)
             json_writer.close_array()
 
         json_writer.close_object()

nomad/processing/data.py

@@ -240,6 +240,9 @@ class Calc(Proc, datamodel.Calc):
             aux_files=list(self.upload_files.calc_files(self.mainfile, with_mainfile=False)))
 
         # persist the repository metadata
+        with utils.timer(logger, 'indexed', step='index'):
+            self.upload_files.metadata.insert(self._parser_backend.metadata())
+
         with utils.timer(logger, 'indexed', step='index'):
             repo_calc = RepoCalc.create_from_backend(
                 self._parser_backend,

tests/processing/test_data.py

@@ -115,6 +115,8 @@ def assert_processing(upload: Upload, mocksearch=None):
             assert repo.basis_set_type is not None
             assert len(repo.aux_files) == 4
 
+        assert upload_files.metadata.get(calc.calc_id) is not None
+
 
 # @pytest.mark.timeout(30)
 def test_processing(uploaded_id, worker, mocksearch, test_user, no_warn):

tests/test_normalizing.py

@@ -50,11 +50,13 @@ def normalized_template_example(parsed_template_example) -> LocalBackend:
 
 
 def assert_normalized(backend):
+    metadata = backend.metadata()['section_repository_info']['section_repository_parserdata']
     count = 0
     for metainfo in backend.metaInfoEnv().infoKindEls():
         if 'section_repository_parserdata' in metainfo.superNames:
             count += 1
             assert backend.get_value(metainfo.name, 0) is not None
+            assert metadata.get(metainfo.name, None) is not None
     assert count > 0
 
 

tests/test_parsing.py

@@ -50,6 +50,16 @@ class TestLocalBackend(object):
     def test_meta_info(self, meta_info, no_warn):
         assert 'section_topology' in meta_info
 
+    def test_metadata(self, backend, no_warn):
+        g_index = backend.openSection('section_calculation_info')
+        assert g_index == 0
+        backend.addValue('calc_id', 't0')
+        backend.closeSection('section_calculation_info', 0)
+        g_index = backend.openSection('section_repository_info')
+        backend.addValue('repository_calc_id', 1)
+        backend.closeSection('section_repository_info', 0)
+        assert json.dumps(backend.metadata()) is not None
+
     def test_section(self, backend, no_warn):
         g_index = backend.openSection('section_run')
         assert g_index == 0