Commit 935e7c23 authored by tlc@void's avatar tlc@void
Browse files

Add support of different versioning in reference

parent fcc3fef7
Pipeline #135697 passed with stages
in 33 minutes and 40 seconds
......@@ -56,6 +56,30 @@ def _default_hash():
return hashlib.new(_hash_method)
def _split_python_definition(definition_with_id: str) -> Tuple[list, Union[str, None]]:
'''
Split a Python type name into names and an optional id.
Example:
mypackage.mysection@myid ==> (['mypackage', 'mysection'], 'myid')
mypackage.mysection ==> (['mypackage', 'mysection'], None)
'''
if '@' not in definition_with_id:
return definition_with_id.split('.'), None
definition_names, definition_id = definition_with_id.split('@')
return definition_names.split('.'), definition_id
def _check_definition_id(target_id, tgt_section: MSectionBound) -> MSectionBound:
'''
Ensure section definition id matches the target id.
'''
if target_id is not None and tgt_section.definition_id != target_id:
raise MetainfoReferenceError(f'Could not resolve {target_id}, id mismatch')
return tgt_section
def to_section_def(section_def: SectionDefOrCls):
'''
Resolves duck-typing for values that are section definitions or section classes to
......@@ -135,7 +159,7 @@ class MEnum(Sequence):
return len(self._list)
class MProxy():
class MProxy:
'''
A placeholder object that acts as reference to a value that is not yet resolved.
......@@ -143,7 +167,7 @@ class MProxy():
The replaced section (or quantity) is identified by a reference. References
are URL strings that identify a section (or quantity).
If a proxy is accessed (i.e. like its proxies counterpart would be accessed), it
If a proxy is accessed (i.e. like its proxies' counterpart would be accessed), it
tries to resolve its reference and access the proxied element. If the reference
cannot be resolved an exception is raised.
......@@ -168,7 +192,7 @@ class MProxy():
The actual algorithm for resolving proxies is in `MSection.m_resolve()`.
Attributes:
m_proxy_value: The reference represented as an URL string.
m_proxy_value: The reference represented as a URL string.
m_proxy_section:
The section context, i.e. the section that this proxy is contained in. This
section will provide the context for resolving the reference. For example,
......@@ -177,8 +201,8 @@ class MProxy():
m_proxy_context:
Optional Context instance. Default is None and the m_context of the m_proxy_section
is used.
m_proxy_quantity:
The quantity defintion. Typically MProxy is used for proxy-ing sections. With
m_proxy_type:
The quantity definition. Typically, MProxy is used for proxy-ing sections. With
this set, the proxy will still act as a normal section proxy, but it will
be used by quantities of type `QuantityReference` to resolve and return
a quantity value.
......@@ -262,12 +286,21 @@ class SectionProxy(MProxy):
if '.' in self.m_proxy_value:
# Try to interpret as python class name
package_name, section_name = self.m_proxy_value.rsplit('.', 1)
python_name, definition_id = _split_python_definition(self.m_proxy_value)
package_name = '.'.join(python_name[:-1])
section_name = python_name[-1]
try:
module = importlib.import_module(package_name)
cls = getattr(module, section_name)
self._set_resolved(cls.m_def)
return self.m_proxy_resolved
if cls.m_def:
if not definition_id or cls.m_def.definition_id == definition_id:
# matches, happy ending
self._set_resolved(cls.m_def)
return self.m_proxy_resolved
# mismatches, use the usual mechanism
return super().m_proxy_resolve()
except Exception:
pass
......@@ -275,17 +308,21 @@ class SectionProxy(MProxy):
if not self.m_proxy_section or self.m_proxy_resolved:
return self.m_proxy_resolved
name_segments = self.m_proxy_value.split('.')
python_name, definition_id = _split_python_definition(self.m_proxy_value)
current = self.m_proxy_section
for name in name_segments:
for name in python_name:
current = self._resolve_name(name, current)
if current is None:
raise MetainfoReferenceError(
f'could not resolve {self.m_proxy_value} from scope {self.m_proxy_section}')
if not definition_id or current.m_def.definition_id == definition_id:
# matches, happy ending
self._set_resolved(current)
return self.m_proxy_resolved
self._set_resolved(current)
return self.m_proxy_resolved
# mismatches, use the usual mechanism
return super().m_proxy_resolve()
class DataType:
......@@ -540,7 +577,7 @@ class _QuantityType(DataType):
@dataclass
class ReferenceURL():
class ReferenceURL:
fragment: str
archive_url: str
url_parts: SplitResult
......@@ -592,12 +629,17 @@ class Reference(DataType):
context_section = proxy.m_proxy_section
if context_section is not None:
context_section = context_section.m_root()
if url.archive_url:
if url.archive_url or '@' in url.fragment:
context = proxy.m_proxy_context
if context is None:
context = context_section.m_context
if not context:
raise MetainfoReferenceError('Proxy with archive url, but no context to resolve it.')
if '@' in url.fragment:
# It's a reference to a section definition
definition, definition_id = f'{url.archive_url}#{url.fragment}'.split('@')
return context.resolve_definition_as_section(definition, definition_id).m_def
context_section = context.resolve_archive_url(url.archive_url)
return self.resolve_fragment(context_section, url.fragment)
......@@ -665,7 +707,10 @@ class Reference(DataType):
# TODO has to deal with URLs, Python qualified names, and Metainfo references
class _SectionReference(Reference):
value_re = re.compile(r'^\w*(\.\w*)*$')
# matches for example
# Python package/module name: nomad.metainfo.section
# Python name + 40 digits id: nomad.metainfo.section@1a2b3c...
value_re = re.compile(r'^\w*(\.\w*)*(@\w{40})?$')
def __init__(self):
super().__init__(None)
......@@ -674,8 +719,14 @@ class _SectionReference(Reference):
def target_section_def(self):
return Section.m_def
def resolve_fragment(self, context_section: 'MSection', fragment: str) -> 'MSection':
def resolve_fragment(self, context_section: 'MSection', fragment_with_id: str) -> 'MSection':
# First, we try to resolve based on definition names
if '@' in fragment_with_id:
fragment, definition_id = fragment_with_id.split('@')
else:
definition_id = None
fragment = fragment_with_id
definitions = None
if isinstance(getattr(context_section, 'definitions', None), Definition):
definitions = getattr(context_section, 'definitions')
......@@ -696,13 +747,13 @@ class _SectionReference(Reference):
if remaining_fragment:
resolved = self.resolve_fragment(content, remaining_fragment)
else:
return content
return _check_definition_id(definition_id, content)
if resolved:
return resolved
return _check_definition_id(definition_id, resolved)
# Resolve regularely as a fallback
return super().resolve_fragment(context_section, fragment)
# Resolve regularly as a fallback
return super().resolve_fragment(context_section, fragment_with_id)
def set_normalize(self, section: 'MSection', quantity_def: 'Quantity', value: Any) -> Any:
if isinstance(value, str) and _SectionReference.value_re.match(value):
......@@ -723,24 +774,28 @@ class _SectionReference(Reference):
def deserialize(self, section: 'MSection', quantity_def: 'Quantity', value: Any) -> Any:
proxy_type = quantity_def.type if quantity_def else SectionReference
if isinstance(value, str) and _SectionReference.value_re.match(value):
# First assume its a python name and try to resolve it.
# First assume it's a python name and try to resolve it.
if '.' in value:
python_name, definition_id = _split_python_definition(value)
package_name = '.'.join(python_name[:-1])
section_name = python_name[-1]
try:
package_name, section_name = value.rsplit('.', 1)
module = importlib.import_module(package_name)
cls = getattr(module, section_name)
if cls:
m_def = getattr(cls, 'm_def')
if m_def:
if m_def and (definition_id is None or m_def.definition_id == definition_id):
# id matches, happy ending
return m_def
except ModuleNotFoundError:
pass
# If its not a python name, we assume its refering to a local metainfo
# definition.
# If it's not a python name or definition id mismatches
# we assume its referring to a local metainfo definition.
return SectionProxy(value, m_proxy_section=section, m_proxy_type=proxy_type)
# Default back to value beeing a URL
# Default back to value being a URL
return MProxy(value, m_proxy_section=section, m_proxy_type=proxy_type)
......@@ -2154,16 +2209,22 @@ class MSection(metaclass=MObjectMeta): # TODO find a way to make this a subclas
'''
return getattr(self, 'm_proxy_resolved', self)
def m_resolve(self, path: str, cls: Type[MSectionBound] = None) -> MSectionBound:
def m_resolve(self, path_with_id: str, cls: Type[MSectionBound] = None) -> MSectionBound:
'''
Resolves the given path or dotted quantity name using this section as context and
returns the sub_section or value.
Arguments:
path: The reference URL. See `MProxy` for details on reference URLs.
path_with_id: The reference URL. See `MProxy` for details on reference URLs.
'''
section: 'MSection' = self
if '@' in path_with_id:
path, target_id = path_with_id.split('@')
else:
target_id = None
path = path_with_id
if path.startswith('/'):
section = section.m_root()
......@@ -2183,7 +2244,7 @@ class MSection(metaclass=MObjectMeta): # TODO find a way to make this a subclas
if isinstance(prop_def, SubSection):
if prop_def.repeats:
if len(path_stack) == 0:
return section.m_get_sub_sections(prop_def) # type: ignore
return _check_definition_id(target_id, section.m_get_sub_sections(prop_def)) # type: ignore
try:
index = int(path_stack.pop())
......@@ -2214,9 +2275,9 @@ class MSection(metaclass=MObjectMeta): # TODO find a way to make this a subclas
raise MetainfoReferenceError(
f'Could not resolve {path}, {prop_name} is not set in {section}')
return section.m_get(prop_def)
return _check_definition_id(target_id, section.m_get(prop_def))
return cast(MSectionBound, section)
return _check_definition_id(target_id, cast(MSectionBound, section))
def m_get_annotations(self, key: Union[str, type], default=None, as_list: bool = False):
'''
......
......@@ -383,7 +383,7 @@ def archive(json_dict):
@pytest.mark.parametrize('definition_id,context,exception_type', [
pytest.param(EntryArchive.m_def.definition_id + 'a', None, MetainfoError, id='wrong_id_no_context'),
pytest.param(EntryArchive.m_def.definition_id + 'a', Context(), NotImplementedError, id='wrong_id_with_context'), ])
pytest.param(EntryArchive.m_def.definition_id[::-1], Context(), NotImplementedError, id='wrong_id_with_context'), ])
def test_archive_with_wrong_id(json_dict, definition_id, context, exception_type):
'''
Test that the archive with wrong id raises the expected exception.
......@@ -392,22 +392,46 @@ def test_archive_with_wrong_id(json_dict, definition_id, context, exception_type
with pytest.raises(exception_type):
EntryArchive.m_from_dict(json_dict, m_context=context)
def test_archive_with_correct_id(json_dict, monkeypatch):
del json_dict['m_def_id']
@pytest.mark.parametrize('m_def,m_def_id', [
pytest.param(None, EntryArchive.m_def.definition_id, id='plain-definition-id'),
pytest.param('nomad.datamodel.EntryArchive', None, id='plain-definition-python-style'),
pytest.param(
'nomad.datamodel.EntryArchive@' + EntryArchive.m_def.definition_id, None,
id='plain-definition-with-correct-id'),
pytest.param(
'nomad.datamodel.EntryArchive@' + EntryArchive.m_def.definition_id[::-1], None,
id='plain-definition-with-wrong-id'),
pytest.param(
'http://my.domain#/placeholder@' + EntryArchive.m_def.definition_id, None,
id='url-definition'),
])
def test_archive_with_id_in_reference(json_dict, m_def, m_def_id, monkeypatch):
'''
Patch Context to return proper section definition to test if the archive is correctly created.
'''
def resolve_definition_as_section(
self, definition: str, definition_id: str): # pylint: disable=unused-argument
def resolve_definition_as_section(self, definition: str, definition_id: str): # pylint: disable=unused-argument
return EntryArchive
monkeypatch.setattr('nomad.metainfo.Context.resolve_definition_as_section', resolve_definition_as_section)
json_dict['m_def_id'] = EntryArchive.m_def.definition_id
if m_def is not None:
json_dict['m_def'] = m_def
if m_def_id is not None:
json_dict['m_def_id'] = m_def_id
archive = MSection.m_from_dict(json_dict, m_context=Context())
assert archive.run is not None
assert len(archive.run) == 1
if 'm_def' in json_dict:
del json_dict['m_def']
if 'm_def_id' in json_dict:
del json_dict['m_def_id']
@pytest.mark.parametrize('required, error', [
pytest.param('include', None, id='include-all'),
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment