Commit 207bff0e authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added metadata from parsing backend to upload_files.

parent b5fddbe2
......@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TextIO, Tuple, List, Any, Callable
from typing import TextIO, Tuple, List, Any, Callable, Iterable
from abc import ABCMeta, abstractmethod
from io import StringIO
import json
......@@ -489,6 +489,27 @@ class LocalBackend(LegacyParserBackend):
else:
json_writer.value(value)
def _obj(self, value: Any, filter: Callable[[str, Any], Any] = None) -> Any:
if isinstance(value, list):
if len(value) == 1 and isinstance(value[0], Section) and not self._delegate.metaInfoEnv().infoKindEl(value[0].name).repeats:
return self._obj(value[0])
else:
return [self._obj(item) for item in value]
elif isinstance(value, Section):
section = value
obj = dict(_name=section.name, _gIndex=section.gIndex)
for name, value in section.items():
if filter is not None:
value = filter(name, value)
if value is not None:
obj[name] = self._obj(value, filter=filter)
return obj
else:
return JSONStreamWriter._json_serializable_value(value)
@property
def status(self) -> ParserStatus:
""" Returns status and potential errors. """
......@@ -499,6 +520,13 @@ class LocalBackend(LegacyParserBackend):
self._errors = None
self._warnings: List[str] = []
def metadata(self, sections: Iterable[str] = ['section_repository_info']) -> dict:
""" Returns an json serializable object with all data of the given sections. """
data = dict(calc_id=self.get_value('calc_id'))
for section in sections:
data[section] = self._obj(self._delegate.results[section])
return data
def write_json(self, out: TextIO, pretty=True, filter: Callable[[str, Any], Any] = None):
"""
Writes the results stored in the backend after parsing in an 'archive'.json
......@@ -516,8 +544,8 @@ class LocalBackend(LegacyParserBackend):
for root_section in ['section_run', 'section_calculation_info', 'section_repository_info']:
json_writer.key(root_section)
json_writer.open_array()
for run in self._delegate.results[root_section]:
LocalBackend._write(json_writer, run, filter=filter)
for section in self._delegate.results[root_section]:
LocalBackend._write(json_writer, section, filter=filter)
json_writer.close_array()
json_writer.close_object()
......
......@@ -240,6 +240,9 @@ class Calc(Proc, datamodel.Calc):
aux_files=list(self.upload_files.calc_files(self.mainfile, with_mainfile=False)))
# persist the repository metadata
with utils.timer(logger, 'indexed', step='index'):
self.upload_files.metadata.insert(self._parser_backend.metadata())
with utils.timer(logger, 'indexed', step='index'):
repo_calc = RepoCalc.create_from_backend(
self._parser_backend,
......
......@@ -115,6 +115,8 @@ def assert_processing(upload: Upload, mocksearch=None):
assert repo.basis_set_type is not None
assert len(repo.aux_files) == 4
assert upload_files.metadata.get(calc.calc_id) is not None
# @pytest.mark.timeout(30)
def test_processing(uploaded_id, worker, mocksearch, test_user, no_warn):
......
......@@ -50,11 +50,13 @@ def normalized_template_example(parsed_template_example) -> LocalBackend:
def assert_normalized(backend):
metadata = backend.metadata()['section_repository_info']['section_repository_parserdata']
count = 0
for metainfo in backend.metaInfoEnv().infoKindEls():
if 'section_repository_parserdata' in metainfo.superNames:
count += 1
assert backend.get_value(metainfo.name, 0) is not None
assert metadata.get(metainfo.name, None) is not None
assert count > 0
......
......@@ -50,6 +50,16 @@ class TestLocalBackend(object):
def test_meta_info(self, meta_info, no_warn):
assert 'section_topology' in meta_info
def test_metadata(self, backend, no_warn):
g_index = backend.openSection('section_calculation_info')
assert g_index == 0
backend.addValue('calc_id', 't0')
backend.closeSection('section_calculation_info', 0)
g_index = backend.openSection('section_repository_info')
backend.addValue('repository_calc_id', 1)
backend.closeSection('section_repository_info', 0)
assert json.dumps(backend.metadata()) is not None
def test_section(self, backend, no_warn):
g_index = backend.openSection('section_run')
assert g_index == 0
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment