Skip to content

GitLab

Commit 60d2111b authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Removed the douple storage of open sections in legacy backend. Removed the... 

Removed the douple storage of open sections in legacy backend. Removed the open/close context mechanism.
parent 828ed9db
Pipeline #79771 passed with stages
in 30 minutes and 59 seconds
Show whitespace changes
Inline Side-by-side
nomad/normalizing/fhiaims.py
View file @ 60d2111b
......@@ -236,8 +236,6 @@ class FhiAimsBaseNormalizer(Normalizer):
# # matrix_hits[key]=matrix_hits[key]+CompareToDefaults(val[AtomIndex],to_compare[i])
# Copen=backend.openContext(context)
# closest_base_int=min(matrix_hits_int, key=matrix_hits_int.get)
# if (matrix_hits_basis[min(matrix_hits_basis, key=matrix_hits_basis.get)] ==0):
......@@ -254,8 +252,6 @@ class FhiAimsBaseNormalizer(Normalizer):
# backend.addValue("basis_set",'custom-'+closest_base_int)
# # print('custom-'+closest_base_int)
# backend.closeContext(context)
# backend.finishedParsingSession("ParseSuccess", None)
# sys.stdout.flush()
# return
......
nomad/parsing/__init__.py
View file @ 60d2111b
......@@ -70,6 +70,6 @@ based on nomad@fairdi's metainfo:
:members:
'''
from nomad.parsing.legacy import AbstractParserBackend, Backend, BackendError, BadContextUri, LegacyParser
from nomad.parsing.legacy import AbstractParserBackend, Backend, BackendError, LegacyParser
from nomad.parsing.parser import Parser, BrokenParser, MissingParser, MatchingParser
from nomad.parsing.artificial import TemplateParser, GenerateRandomParser, ChaosParser, EmptyParser
nomad/parsing/legacy.py
View file @ 60d2111b
......@@ -18,7 +18,7 @@ new nomad@fairdi infrastructure. This covers aspects like the new metainfo, a un
wrapper for parsers, parser logging, and a parser backend.
'''
from typing import Dict, List, Union, Any, Tuple, Type, cast
from typing import List, Union, Any, Tuple, Type, cast
from abc import ABCMeta, abstractmethod
import importlib
import os.path
......@@ -30,7 +30,7 @@ import sys
from nomad import utils, datamodel, config
from nomad.metainfo import (
SubSection, Quantity, Section, Reference, MResource, MSection, MSectionBound, Property)
SubSection, Quantity, Section, SectionProxy, Reference, MResource, MSection, MSectionBound, Property)
from nomad.metainfo.legacy import (
LegacyMetainfoEnvironment, python_package_mapping, normalize_name)
......@@ -41,10 +41,6 @@ class BackendError(Exception):
pass
class BadContextUri(Exception):
pass
class AbstractParserBackend(metaclass=ABCMeta):
'''
This ABS provides the parser backend interface used by the NOMAD-coe parsers.
......@@ -71,16 +67,6 @@ class AbstractParserBackend(metaclass=ABCMeta):
''' Called when the parsing finishes. '''
pass
@abstractmethod
def openContext(self, contextUri: str):
''' Open existing archive data to introduce new data into an existing section. '''
pass
@abstractmethod
def closeContext(self, contextUri: str):
''' Close priorly opened existing archive data again. '''
pass
@abstractmethod
def openSection(self, metaName, parent_index=-1):
''' Opens a new section and returns its new unique gIndex. '''
......@@ -229,7 +215,6 @@ class Backend(AbstractParserBackend):
self.entry_archive = datamodel.EntryArchive()
self.resource.add(self.entry_archive)
self.__open_sections: Dict[Tuple[Section, int], MSection] = {}
self.strict = False # TODO
self.reset_status()
......@@ -238,15 +223,31 @@ class Backend(AbstractParserBackend):
# self._known_attributes = ['results']
# self.fileOut = io.StringIO()
def __open_section(self, section_def: Union[Section, SectionProxy], index: int = -1) -> MSection:
''' Returns the last opened section with the given parent index. '''
sections = self.resource.all(section_def.section_cls)
if sections is None:
raise KeyError(
'section %s with given parent index %s does not exist' % (section_def.name, index))
if index == -1:
return sections[-1]
for section in reversed(sections):
if section.m_parent_index == index:
return section
raise KeyError(
'section %s with given parent index %s does not exist (B)' % (section_def.name, index))
def __getitem__(self, key):
property_def = self.resolve_definition(key, Property)
section_def = property_def.m_parent
if section_def.extends_base_section:
section_def = section_def.base_sections[0]
section = self.__open_sections.get((section_def, -1), None)
if not section:
section = self.open_sections[(section_def, -1)]
section = self.__open_section(section_def)
if isinstance(property_def, Quantity):
return section.m_get(property_def)
......@@ -262,69 +263,6 @@ class Backend(AbstractParserBackend):
return self.env.resolve_definition(
normalize_name(name), section_cls, Backend.filter_legacy_defs)
def resolve_context(self, context_uri: str):
path = context_uri.strip('/').split('/')
path.reverse()
current = None
while len(path) > 0:
section = path.pop()
if len(path) == 0:
raise BadContextUri(context_uri)
index = int(path.pop())
if current is None:
section_def = self.resolve_definition(section, Section)
if section_def is None:
raise BadContextUri(context_uri)
i = 0
for content in self.entry_archive.m_contents():
if content.m_follows(section_def):
if i == index:
current = content
break
i += 1
else:
sub_section_def = self.resolve_definition(section, SubSection)
if sub_section_def is None:
raise BadContextUri(context_uri)
current = current.m_get_sub_section(sub_section_def, index)
if current is None:
raise BadContextUri(context_uri)
return current
def openContext(self, context_uri: str):
''' Open existing archive data to introduce new data into an existing section. '''
section = self.resolve_context(context_uri)
self.__open(section)
def closeContext(self, context_uri: str):
''' Close priorly opened existing archive data again. '''
section = self.resolve_context(context_uri)
self.__close(section)
@property
def open_sections(self):
for section in self.entry_archive.m_all_contents():
self.__open(section)
return self.__open_sections
def __open(self, section):
if section.m_parent_index != -1:
self.__open_sections[(section.m_def, section.m_parent_index)] = section
# here -1 meaning the last opened section
self.__open_sections[(section.m_def, -1)] = section
def __close(self, section):
# TODO
pass
# if section.m_parent_index != -1 and self.__open_sections.get((section.m_def, -1)) == section:
# del(self.__open_sections[(section.m_def, -1)])
# del(self.__open_sections[(section.m_def, section.m_parent_index)])
def openSection(self, name, parent_index: int = -1, return_section=False):
'''
It will assume that there is a sub-section def with the given name.
......@@ -349,11 +287,9 @@ class Backend(AbstractParserBackend):
if parent_section_def.extends_base_section:
parent_section_def = parent_section_def.base_sections[0]
parent = self.__open_sections[(parent_section_def, parent_index)]
parent = self.__open_section(parent_section_def, parent_index)
section = parent.m_create(section_def.section_cls, sub_section_def)
self.__open(section)
if return_section:
return section
return section.m_parent_index
......@@ -366,18 +302,12 @@ class Backend(AbstractParserBackend):
if section_def.extends_base_section:
section_def = section_def.base_sections[0]
section = self.__open_sections.get((section_def, g_index), None)
if not section:
section = self.open_sections[(section_def, g_index)]
section = self.__open_section(section_def, g_index)
return section, quantity_def
def closeSection(self, name, g_index):
# TODO
if self.strict:
section_def = self.resolve_definition(name, Section)
section = self.__open_sections[(section_def, g_index)]
self.__close(section)
pass
def openNonOverlappingSection(self, metaName):
return self.openSection(metaName)
......@@ -404,14 +334,11 @@ class Backend(AbstractParserBackend):
section, quantity_def = self.get_open_section_for_quantity(name, g_index)
if isinstance(quantity_def.type, Reference):
# quantity is a reference
possible_targets = self.resource.all(quantity_def.type.target_section_def.section_cls)
referenced_target = None
for target in possible_targets:
if target.m_parent_index == value:
referenced_target = target
if referenced_target is None:
raise BackendError('There is not section for the given reference index')
try:
referenced_target = self.__open_section(
quantity_def.type.target_section_def, value)
except KeyError as e:
raise BackendError(str(e))
value = referenced_target
......@@ -444,15 +371,12 @@ class Backend(AbstractParserBackend):
section, quantity_def = self.get_open_section_for_quantity(name, gIndex)
if isinstance(quantity_def.type, Reference):
# quantity is a reference
possible_targets = self.resource.all(quantity_def.type.target_section_def.section_cls)
resolved_values = []
for value in values:
referenced_target = None
for target in possible_targets:
if target.m_parent_index == value:
referenced_target = target
if referenced_target is None:
# quantity is a reference
try:
referenced_target = self.__open_section(quantity_def.type.target_section_def, value)
except KeyError:
raise BackendError('There is not section for the given reference index')
resolved_values.append(referenced_target)
......
tests/test_parsing.py
View file @ 60d2111b
......@@ -20,7 +20,7 @@ import os
from shutil import copyfile
from nomad import utils, files, datamodel
from nomad.parsing import BrokenParser, BadContextUri, Backend
from nomad.parsing import BrokenParser, Backend
from nomad.parsing.parsers import parser_dict, match_parser
from nomad.app import dump_json
from nomad.metainfo import MSection
......@@ -225,62 +225,6 @@ class TestBackend(object):
assert len(run['section_single_configuration_calculation'][0].section_dos) == 1
assert len(run['section_single_configuration_calculation'][1].section_dos) == 0
def test_context(self, backend: Backend, no_warn):
backend.openSection('section_run')
backend.openSection('section_method')
backend.closeSection('section_method', -1)
backend.closeSection('section_run', -1)
backend.openSection('section_run')
backend.closeSection('section_run', -1)
backend.openContext('/section_run/0')
backend.addValue('program_name', 't1')
backend.closeContext('/section_run/0')
backend.openContext('/section_run/1')
backend.addValue('program_name', 't2')
backend.closeContext('/section_run/1')
backend.openContext('/section_run/0/section_method/0')
backend.closeContext('/section_run/0/section_method/0')
from nomad.datamodel.metainfo.public import section_run
runs = backend.resource.all(section_run)
assert runs[0]['program_name'] == 't1'
assert runs[1]['program_name'] == 't2'
def test_multi_context(self, backend: Backend, no_warn):
backend.openSection('section_run')
backend.closeSection('section_run', -1)
backend.openContext('/section_run/0')
backend.openSection('section_method')
backend.closeSection('section_method', -1)
backend.closeContext('/section_run/0')
backend.openContext('/section_run/0')
backend.openSection('section_method')
backend.closeSection('section_method', -1)
backend.closeContext('/section_run/0')
from nomad.datamodel.metainfo.public import section_run
runs = backend.resource.all(section_run)
assert len(runs[0].section_method) == 2
def test_bad_context(self, backend: Backend, no_warn):
try:
backend.openContext('section_run/0')
assert False
except BadContextUri:
pass
try:
backend.openContext('dsfds')
assert False
except BadContextUri:
pass
def create_reference(data, pretty):
if (pretty):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment