Commit 60d2111b authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Removed the douple storage of open sections in legacy backend. Removed the...

Removed the douple storage of open sections in legacy backend. Removed the open/close context mechanism.
parent 828ed9db
Pipeline #79771 passed with stages
in 30 minutes and 59 seconds
...@@ -236,8 +236,6 @@ class FhiAimsBaseNormalizer(Normalizer): ...@@ -236,8 +236,6 @@ class FhiAimsBaseNormalizer(Normalizer):
# # matrix_hits[key]=matrix_hits[key]+CompareToDefaults(val[AtomIndex],to_compare[i]) # # matrix_hits[key]=matrix_hits[key]+CompareToDefaults(val[AtomIndex],to_compare[i])
# Copen=backend.openContext(context)
# closest_base_int=min(matrix_hits_int, key=matrix_hits_int.get) # closest_base_int=min(matrix_hits_int, key=matrix_hits_int.get)
# if (matrix_hits_basis[min(matrix_hits_basis, key=matrix_hits_basis.get)] ==0): # if (matrix_hits_basis[min(matrix_hits_basis, key=matrix_hits_basis.get)] ==0):
...@@ -254,8 +252,6 @@ class FhiAimsBaseNormalizer(Normalizer): ...@@ -254,8 +252,6 @@ class FhiAimsBaseNormalizer(Normalizer):
# backend.addValue("basis_set",'custom-'+closest_base_int) # backend.addValue("basis_set",'custom-'+closest_base_int)
# # print('custom-'+closest_base_int) # # print('custom-'+closest_base_int)
# backend.closeContext(context)
# backend.finishedParsingSession("ParseSuccess", None) # backend.finishedParsingSession("ParseSuccess", None)
# sys.stdout.flush() # sys.stdout.flush()
# return # return
......
...@@ -70,6 +70,6 @@ based on nomad@fairdi's metainfo: ...@@ -70,6 +70,6 @@ based on nomad@fairdi's metainfo:
:members: :members:
''' '''
from nomad.parsing.legacy import AbstractParserBackend, Backend, BackendError, BadContextUri, LegacyParser from nomad.parsing.legacy import AbstractParserBackend, Backend, BackendError, LegacyParser
from nomad.parsing.parser import Parser, BrokenParser, MissingParser, MatchingParser from nomad.parsing.parser import Parser, BrokenParser, MissingParser, MatchingParser
from nomad.parsing.artificial import TemplateParser, GenerateRandomParser, ChaosParser, EmptyParser from nomad.parsing.artificial import TemplateParser, GenerateRandomParser, ChaosParser, EmptyParser
...@@ -18,7 +18,7 @@ new nomad@fairdi infrastructure. This covers aspects like the new metainfo, a un ...@@ -18,7 +18,7 @@ new nomad@fairdi infrastructure. This covers aspects like the new metainfo, a un
wrapper for parsers, parser logging, and a parser backend. wrapper for parsers, parser logging, and a parser backend.
''' '''
from typing import Dict, List, Union, Any, Tuple, Type, cast from typing import List, Union, Any, Tuple, Type, cast
from abc import ABCMeta, abstractmethod from abc import ABCMeta, abstractmethod
import importlib import importlib
import os.path import os.path
...@@ -30,7 +30,7 @@ import sys ...@@ -30,7 +30,7 @@ import sys
from nomad import utils, datamodel, config from nomad import utils, datamodel, config
from nomad.metainfo import ( from nomad.metainfo import (
SubSection, Quantity, Section, Reference, MResource, MSection, MSectionBound, Property) SubSection, Quantity, Section, SectionProxy, Reference, MResource, MSection, MSectionBound, Property)
from nomad.metainfo.legacy import ( from nomad.metainfo.legacy import (
LegacyMetainfoEnvironment, python_package_mapping, normalize_name) LegacyMetainfoEnvironment, python_package_mapping, normalize_name)
...@@ -41,10 +41,6 @@ class BackendError(Exception): ...@@ -41,10 +41,6 @@ class BackendError(Exception):
pass pass
class BadContextUri(Exception):
pass
class AbstractParserBackend(metaclass=ABCMeta): class AbstractParserBackend(metaclass=ABCMeta):
''' '''
This ABS provides the parser backend interface used by the NOMAD-coe parsers. This ABS provides the parser backend interface used by the NOMAD-coe parsers.
...@@ -71,16 +67,6 @@ class AbstractParserBackend(metaclass=ABCMeta): ...@@ -71,16 +67,6 @@ class AbstractParserBackend(metaclass=ABCMeta):
''' Called when the parsing finishes. ''' ''' Called when the parsing finishes. '''
pass pass
@abstractmethod
def openContext(self, contextUri: str):
''' Open existing archive data to introduce new data into an existing section. '''
pass
@abstractmethod
def closeContext(self, contextUri: str):
''' Close priorly opened existing archive data again. '''
pass
@abstractmethod @abstractmethod
def openSection(self, metaName, parent_index=-1): def openSection(self, metaName, parent_index=-1):
''' Opens a new section and returns its new unique gIndex. ''' ''' Opens a new section and returns its new unique gIndex. '''
...@@ -229,7 +215,6 @@ class Backend(AbstractParserBackend): ...@@ -229,7 +215,6 @@ class Backend(AbstractParserBackend):
self.entry_archive = datamodel.EntryArchive() self.entry_archive = datamodel.EntryArchive()
self.resource.add(self.entry_archive) self.resource.add(self.entry_archive)
self.__open_sections: Dict[Tuple[Section, int], MSection] = {}
self.strict = False # TODO self.strict = False # TODO
self.reset_status() self.reset_status()
...@@ -238,15 +223,31 @@ class Backend(AbstractParserBackend): ...@@ -238,15 +223,31 @@ class Backend(AbstractParserBackend):
# self._known_attributes = ['results'] # self._known_attributes = ['results']
# self.fileOut = io.StringIO() # self.fileOut = io.StringIO()
def __open_section(self, section_def: Union[Section, SectionProxy], index: int = -1) -> MSection:
''' Returns the last opened section with the given parent index. '''
sections = self.resource.all(section_def.section_cls)
if sections is None:
raise KeyError(
'section %s with given parent index %s does not exist' % (section_def.name, index))
if index == -1:
return sections[-1]
for section in reversed(sections):
if section.m_parent_index == index:
return section
raise KeyError(
'section %s with given parent index %s does not exist (B)' % (section_def.name, index))
def __getitem__(self, key): def __getitem__(self, key):
property_def = self.resolve_definition(key, Property) property_def = self.resolve_definition(key, Property)
section_def = property_def.m_parent section_def = property_def.m_parent
if section_def.extends_base_section: if section_def.extends_base_section:
section_def = section_def.base_sections[0] section_def = section_def.base_sections[0]
section = self.__open_sections.get((section_def, -1), None) section = self.__open_section(section_def)
if not section:
section = self.open_sections[(section_def, -1)]
if isinstance(property_def, Quantity): if isinstance(property_def, Quantity):
return section.m_get(property_def) return section.m_get(property_def)
...@@ -262,69 +263,6 @@ class Backend(AbstractParserBackend): ...@@ -262,69 +263,6 @@ class Backend(AbstractParserBackend):
return self.env.resolve_definition( return self.env.resolve_definition(
normalize_name(name), section_cls, Backend.filter_legacy_defs) normalize_name(name), section_cls, Backend.filter_legacy_defs)
def resolve_context(self, context_uri: str):
path = context_uri.strip('/').split('/')
path.reverse()
current = None
while len(path) > 0:
section = path.pop()
if len(path) == 0:
raise BadContextUri(context_uri)
index = int(path.pop())
if current is None:
section_def = self.resolve_definition(section, Section)
if section_def is None:
raise BadContextUri(context_uri)
i = 0
for content in self.entry_archive.m_contents():
if content.m_follows(section_def):
if i == index:
current = content
break
i += 1
else:
sub_section_def = self.resolve_definition(section, SubSection)
if sub_section_def is None:
raise BadContextUri(context_uri)
current = current.m_get_sub_section(sub_section_def, index)
if current is None:
raise BadContextUri(context_uri)
return current
def openContext(self, context_uri: str):
''' Open existing archive data to introduce new data into an existing section. '''
section = self.resolve_context(context_uri)
self.__open(section)
def closeContext(self, context_uri: str):
''' Close priorly opened existing archive data again. '''
section = self.resolve_context(context_uri)
self.__close(section)
@property
def open_sections(self):
for section in self.entry_archive.m_all_contents():
self.__open(section)
return self.__open_sections
def __open(self, section):
if section.m_parent_index != -1:
self.__open_sections[(section.m_def, section.m_parent_index)] = section
# here -1 meaning the last opened section
self.__open_sections[(section.m_def, -1)] = section
def __close(self, section):
# TODO
pass
# if section.m_parent_index != -1 and self.__open_sections.get((section.m_def, -1)) == section:
# del(self.__open_sections[(section.m_def, -1)])
# del(self.__open_sections[(section.m_def, section.m_parent_index)])
def openSection(self, name, parent_index: int = -1, return_section=False): def openSection(self, name, parent_index: int = -1, return_section=False):
''' '''
It will assume that there is a sub-section def with the given name. It will assume that there is a sub-section def with the given name.
...@@ -349,11 +287,9 @@ class Backend(AbstractParserBackend): ...@@ -349,11 +287,9 @@ class Backend(AbstractParserBackend):
if parent_section_def.extends_base_section: if parent_section_def.extends_base_section:
parent_section_def = parent_section_def.base_sections[0] parent_section_def = parent_section_def.base_sections[0]
parent = self.__open_sections[(parent_section_def, parent_index)] parent = self.__open_section(parent_section_def, parent_index)
section = parent.m_create(section_def.section_cls, sub_section_def) section = parent.m_create(section_def.section_cls, sub_section_def)
self.__open(section)
if return_section: if return_section:
return section return section
return section.m_parent_index return section.m_parent_index
...@@ -366,18 +302,12 @@ class Backend(AbstractParserBackend): ...@@ -366,18 +302,12 @@ class Backend(AbstractParserBackend):
if section_def.extends_base_section: if section_def.extends_base_section:
section_def = section_def.base_sections[0] section_def = section_def.base_sections[0]
section = self.__open_sections.get((section_def, g_index), None) section = self.__open_section(section_def, g_index)
if not section:
section = self.open_sections[(section_def, g_index)]
return section, quantity_def return section, quantity_def
def closeSection(self, name, g_index): def closeSection(self, name, g_index):
# TODO pass
if self.strict:
section_def = self.resolve_definition(name, Section)
section = self.__open_sections[(section_def, g_index)]
self.__close(section)
def openNonOverlappingSection(self, metaName): def openNonOverlappingSection(self, metaName):
return self.openSection(metaName) return self.openSection(metaName)
...@@ -404,14 +334,11 @@ class Backend(AbstractParserBackend): ...@@ -404,14 +334,11 @@ class Backend(AbstractParserBackend):
section, quantity_def = self.get_open_section_for_quantity(name, g_index) section, quantity_def = self.get_open_section_for_quantity(name, g_index)
if isinstance(quantity_def.type, Reference): if isinstance(quantity_def.type, Reference):
# quantity is a reference # quantity is a reference
possible_targets = self.resource.all(quantity_def.type.target_section_def.section_cls) try:
referenced_target = None referenced_target = self.__open_section(
for target in possible_targets: quantity_def.type.target_section_def, value)
if target.m_parent_index == value: except KeyError as e:
referenced_target = target raise BackendError(str(e))
if referenced_target is None:
raise BackendError('There is not section for the given reference index')
value = referenced_target value = referenced_target
...@@ -444,15 +371,12 @@ class Backend(AbstractParserBackend): ...@@ -444,15 +371,12 @@ class Backend(AbstractParserBackend):
section, quantity_def = self.get_open_section_for_quantity(name, gIndex) section, quantity_def = self.get_open_section_for_quantity(name, gIndex)
if isinstance(quantity_def.type, Reference): if isinstance(quantity_def.type, Reference):
# quantity is a reference # quantity is a reference
possible_targets = self.resource.all(quantity_def.type.target_section_def.section_cls)
resolved_values = [] resolved_values = []
for value in values: for value in values:
referenced_target = None # quantity is a reference
for target in possible_targets: try:
if target.m_parent_index == value: referenced_target = self.__open_section(quantity_def.type.target_section_def, value)
referenced_target = target except KeyError:
if referenced_target is None:
raise BackendError('There is not section for the given reference index') raise BackendError('There is not section for the given reference index')
resolved_values.append(referenced_target) resolved_values.append(referenced_target)
......
...@@ -20,7 +20,7 @@ import os ...@@ -20,7 +20,7 @@ import os
from shutil import copyfile from shutil import copyfile
from nomad import utils, files, datamodel from nomad import utils, files, datamodel
from nomad.parsing import BrokenParser, BadContextUri, Backend from nomad.parsing import BrokenParser, Backend
from nomad.parsing.parsers import parser_dict, match_parser from nomad.parsing.parsers import parser_dict, match_parser
from nomad.app import dump_json from nomad.app import dump_json
from nomad.metainfo import MSection from nomad.metainfo import MSection
...@@ -225,62 +225,6 @@ class TestBackend(object): ...@@ -225,62 +225,6 @@ class TestBackend(object):
assert len(run['section_single_configuration_calculation'][0].section_dos) == 1 assert len(run['section_single_configuration_calculation'][0].section_dos) == 1
assert len(run['section_single_configuration_calculation'][1].section_dos) == 0 assert len(run['section_single_configuration_calculation'][1].section_dos) == 0
def test_context(self, backend: Backend, no_warn):
backend.openSection('section_run')
backend.openSection('section_method')
backend.closeSection('section_method', -1)
backend.closeSection('section_run', -1)
backend.openSection('section_run')
backend.closeSection('section_run', -1)
backend.openContext('/section_run/0')
backend.addValue('program_name', 't1')
backend.closeContext('/section_run/0')
backend.openContext('/section_run/1')
backend.addValue('program_name', 't2')
backend.closeContext('/section_run/1')
backend.openContext('/section_run/0/section_method/0')
backend.closeContext('/section_run/0/section_method/0')
from nomad.datamodel.metainfo.public import section_run
runs = backend.resource.all(section_run)
assert runs[0]['program_name'] == 't1'
assert runs[1]['program_name'] == 't2'
def test_multi_context(self, backend: Backend, no_warn):
backend.openSection('section_run')
backend.closeSection('section_run', -1)
backend.openContext('/section_run/0')
backend.openSection('section_method')
backend.closeSection('section_method', -1)
backend.closeContext('/section_run/0')
backend.openContext('/section_run/0')
backend.openSection('section_method')
backend.closeSection('section_method', -1)
backend.closeContext('/section_run/0')
from nomad.datamodel.metainfo.public import section_run
runs = backend.resource.all(section_run)
assert len(runs[0].section_method) == 2
def test_bad_context(self, backend: Backend, no_warn):
try:
backend.openContext('section_run/0')
assert False
except BadContextUri:
pass
try:
backend.openContext('dsfds')
assert False
except BadContextUri:
pass
def create_reference(data, pretty): def create_reference(data, pretty):
if (pretty): if (pretty):
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment