diff --git a/nomad/app/v1/models/graph/graph_models.py b/nomad/app/v1/models/graph/graph_models.py
index b3c833601ba9b1065c1cdf0df93a94110a658222..ff4b2151c21999bbcf866d2c235e589bbba52142 100644
--- a/nomad/app/v1/models/graph/graph_models.py
+++ b/nomad/app/v1/models/graph/graph_models.py
@@ -21,7 +21,12 @@ from typing import Optional, List, Union, Any, Literal
from pydantic import BaseModel, Field, Extra
from nomad.metainfo import Package
-from nomad.graph.model import RequestConfig, DatasetQuery
+from nomad.graph.model import (
+ RequestConfig,
+ DatasetQuery,
+ MetainfoQuery,
+ MetainfoPagination,
+)
from nomad.metainfo.pydantic_extension import PydanticModel
from nomad.datamodel.data import User as UserModel
from nomad.app.v1.models.models import Metadata, MetadataResponse
@@ -217,7 +222,20 @@ class GraphDatasets(BaseModel):
m_children: GraphDataset
+class MetainfoRequestOptions(BaseModel):
+ pagination: Optional[MetainfoPagination]
+ query: Optional[MetainfoQuery]
+
+
+class MetainfoResponseOptions(BaseModel):
+ pagination: Optional[PaginationResponse]
+ query: Optional[MetainfoQuery]
+
+
class GraphMetainfo(BaseModel):
+ m_request: MetainfoRequestOptions
+ m_response: MetainfoResponseOptions
+ m_errors: List[Error]
m_children: MSection
diff --git a/nomad/graph/graph_reader.py b/nomad/graph/graph_reader.py
index eae2bd8aed531a0fe4476e16ff8a1be50fc3e28b..29ded7015f26658106c778373e182217a1daa4b4 100644
--- a/nomad/graph/graph_reader.py
+++ b/nomad/graph/graph_reader.py
@@ -24,9 +24,12 @@ import functools
import itertools
import os
import re
+from collections.abc import Iterator, AsyncIterator
+from threading import Lock
from typing import Any, Callable, Type
import orjson
+from cachetools import TTLCache
from fastapi import HTTPException
from mongoengine import Q
@@ -49,6 +52,9 @@ from nomad.app.v1.routers.uploads import (
RawDirPagination,
)
from nomad.archive import ArchiveList, ArchiveDict, to_json
+from nomad.datamodel import ServerContext, User, EntryArchive, Dataset
+from nomad.datamodel.util import parse_path
+from nomad.files import UploadFiles, RawPathInfo
from nomad.graph.model import (
RequestConfig,
DefinitionType,
@@ -56,10 +62,9 @@ from nomad.graph.model import (
ResolveType,
DatasetQuery,
EntryQuery,
+ MetainfoQuery,
+ MetainfoPagination,
)
-from nomad.datamodel import ServerContext, User, EntryArchive, Dataset
-from nomad.datamodel.util import parse_path
-from nomad.files import UploadFiles, RawPathInfo
from nomad.metainfo import (
SubSection,
QuantityReference,
@@ -70,7 +75,6 @@ from nomad.metainfo import (
Definition,
Section,
)
-
from nomad.metainfo.data_type import Any as AnyType, JSON
from nomad.metainfo.util import split_python_definition, MSubSectionList
from nomad.processing import Entry, Upload, ProcessStatus
@@ -396,7 +400,14 @@ def _to_response_config(config: RequestConfig, exclude: list = None, **kwargs):
return response_config
-async def _populate_result(container_root: dict, path: list, value, *, path_like=False):
+async def _populate_result(
+ container_root: dict,
+ path: list,
+ value,
+ *,
+ path_like=False,
+ overwrite_existing_str=False,
+):
"""
For the given path and the root of the target container, populate the value.
@@ -476,6 +487,8 @@ async def _populate_result(container_root: dict, path: list, value, *, path_like
assert isinstance(key_or_index, int)
if target_container[key_or_index] is None:
target_container[key_or_index] = new_value
+ elif isinstance(target_container[key_or_index], str) and overwrite_existing_str:
+ target_container[key_or_index] = new_value
elif isinstance(new_value, dict):
_merge_dict(target_container[key_or_index], new_value)
elif isinstance(new_value, list):
@@ -484,7 +497,12 @@ async def _populate_result(container_root: dict, path: list, value, *, path_like
target_container[key_or_index] = new_value
elif isinstance(target_container, dict):
assert isinstance(key_or_index, str)
- if isinstance(new_value, dict):
+ if (
+ isinstance(target_container.get(key_or_index, None), str)
+ and overwrite_existing_str
+ ):
+ target_container[key_or_index] = new_value
+ elif isinstance(new_value, dict):
target_container.setdefault(key_or_index, {})
_merge_dict(target_container[key_or_index], new_value)
elif isinstance(new_value, list):
@@ -571,6 +589,9 @@ def _normalise_required(
elif name == Token.SEARCH:
reader_type = ElasticSearchReader
can_query = True
+ elif name == Token.METAINFO:
+ reader_type = MetainfoBrowser
+ can_query = True
elif name == Token.RAW or name == Token.MAINFILE:
reader_type = FileSystemReader
elif name == Token.ARCHIVE:
@@ -705,6 +726,10 @@ def _normalise_index(index: tuple | None, length: int) -> range:
return range(_bound(start), _bound(end) + 1)
+def _unwrap_subsection(target):
+ return target.sub_section.m_resolved() if isinstance(target, SubSection) else target
+
+
class GeneralReader:
# controls the name of configuration
# it will be extracted from the query dict to generate the configuration object
@@ -1046,6 +1071,111 @@ class GeneralReader:
return asyncio.run(self.read(*args, **kwargs))
+# module level TTL cache for caching packages
+__lock_pool = Lock()
+__package_pool = TTLCache(maxsize=128, ttl=300)
+
+
+def _fetch_package(key: str) -> Package:
+ with __lock_pool:
+ return __package_pool.get(key, None)
+
+
+def _cache_package(key: str, package: Package):
+ with __lock_pool:
+ __package_pool[key] = package
+
+
+class ArchiveLikeReader(GeneralReader):
+ """
+ An abstract class for `ArchiveReader` and `DefinitionReader`.
+ """
+
+ # noinspection PyUnusedLocal
+ async def _retrieve_definition(
+ self,
+ m_def: str | None,
+ m_def_id: str | None = None,
+ node: GraphNode | None = None,
+ ):
+ """
+ Retrieve a definition from an archive.
+ The definition is identified by `m_def` and/or `m_def_id`.
+ It could be a local definition that is defined in the `definitions` section.
+ In this case, we initialise a new `Package` object and resolve the path.
+ It could also be a reference to another entry in another upload.
+ In this case, we need to load the archive.
+
+ todo: more flexible definition retrieval, accounting for definition id, mismatches, etc.
+ """
+
+ async def __resolve_definition_in_archive(
+ _root,
+ _path_stack: list,
+ _upload_id: str = None,
+ _entry_id: str = None,
+ ):
+ cache_key: str = f'{_upload_id}:{_entry_id}'
+
+ custom_package: Package | None = _fetch_package(cache_key)
+ if custom_package is None:
+ custom_package = Package.m_from_dict(
+ await async_to_json(await goto_child(_root, 'definitions')),
+ m_context=ServerContext(
+ get_upload_with_read_access(
+ _upload_id, self.user, include_others=True
+ )
+ ),
+ )
+ # package loaded in this way does not have an attached archive
+ # we manually set the upload_id and entry_id so that
+ # correct references can be generated in the corresponding method
+ custom_package.entry_id = _entry_id
+ custom_package.upload_id = _upload_id
+ custom_package.init_metainfo()
+ if (
+ upload := Upload.objects(upload_id=_upload_id).first()
+ ) is not None and upload.published:
+ _cache_package(cache_key, custom_package)
+
+ return custom_package.m_resolve_path(_path_stack)
+
+ if m_def is not None:
+ if m_def.startswith(('#/', '/')):
+ # appears to be a local definition
+ return await __resolve_definition_in_archive(
+ node.archive_root,
+ [v for v in m_def.split('/') if v not in ('', '#', 'definitions')],
+ await goto_child(node.archive_root, ['metadata', 'upload_id']),
+ await goto_child(node.archive_root, ['metadata', 'entry_id']),
+ )
+ # todo: !!!need to unify different formats!!!
+ # check if m_def matches the pattern 'entry_id:example_id.example_section.example_quantity'
+ if m_def.startswith('entry_id:'):
+ tokens = m_def[9:].split('.')
+ entry_id = tokens.pop(0)
+ entry_record = Entry.objects(entry_id=entry_id).first()
+ upload_id = entry_record.upload_id
+ if (
+ cached_package := _fetch_package(f'{upload_id}:{entry_id}')
+ ) is not None: # early fetch to avoid loading archive from disk
+ return cached_package.m_resolve_path(tokens)
+ archive = self.load_archive(upload_id, entry_id)
+ return await __resolve_definition_in_archive(
+ archive, tokens, upload_id, entry_id
+ )
+
+ # further consider when only m_def_id is given, etc.
+
+ # this is not likely to be reached
+ # it does not work anyway
+ proxy = SectionReference().normalize(m_def)
+ proxy.m_proxy_context = ServerContext(
+ get_upload_with_read_access(node.upload_id, self.user, include_others=True)
+ )
+ return proxy.section_cls.m_def
+
+
class MongoReader(GeneralReader):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -1341,7 +1471,9 @@ class MongoReader(GeneralReader):
if key in (GeneralReader.__CONFIG__, GeneralReader.__WILDCARD__):
continue
- async def offload_read(reader_cls, *args, read_list=False):
+ async def offload_read(
+ reader_cls: Type[GeneralReader], *args, read_list=False
+ ):
try:
with reader_cls(value, **offload_pack) as reader:
await _populate_result(
@@ -1386,6 +1518,11 @@ class MongoReader(GeneralReader):
await offload_read(EntryReader, node.archive['entry_id'])
continue
+ if key == Token.METAINFO and self.__class__ is MongoReader:
+ # hitting the bottom of the current scope
+ await offload_read(MetainfoBrowser)
+ continue
+
if isinstance(node.archive, dict) and isinstance(value, dict):
# treat it as a normal key
# and handle in applying resolver if it is a leaf node
@@ -2146,7 +2283,7 @@ class FileSystemReader(GeneralReader):
return {}
-class ArchiveReader(GeneralReader):
+class ArchiveReader(ArchiveLikeReader):
"""
This class provides functionalities to read an archive with the required fields.
A sample query will look like the following.
@@ -2181,7 +2318,6 @@ class ArchiveReader(GeneralReader):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
- self.package_pool: dict = {}
@staticmethod
def __if_strip(node: GraphNode, config: RequestConfig, *, depth_check: bool = True):
@@ -2601,70 +2737,6 @@ class ArchiveReader(GeneralReader):
resolved_node.replace(definition=target.m_resolved()), config
)
- # noinspection PyUnusedLocal
- async def _retrieve_definition(
- self, m_def: str | None, m_def_id: str | None, node: GraphNode
- ):
- # todo: more flexible definition retrieval, accounting for definition id, mismatches, etc.
- context = ServerContext(
- get_upload_with_read_access(node.upload_id, self.user, include_others=True)
- )
-
- def __resolve_definition_in_archive(
- _root: dict,
- _path_stack: list,
- _upload_id: str = None,
- _entry_id: str = None,
- ):
- cache_key = f'{_upload_id}:{_entry_id}'
-
- if cache_key not in self.package_pool:
- custom_def_package: Package = Package.m_from_dict(
- _root, m_context=context
- )
- # package loaded in this way does not have an attached archive
- # we manually set the upload_id and entry_id so that
- # correct references can be generated in the corresponding method
- custom_def_package.entry_id = _entry_id
- custom_def_package.upload_id = _upload_id
- custom_def_package.init_metainfo()
- self.package_pool[cache_key] = custom_def_package
-
- return self.package_pool[cache_key].m_resolve_path(_path_stack)
-
- if m_def is not None:
- if m_def.startswith(('#/', '/')):
- # appears to be a local definition
- return __resolve_definition_in_archive(
- await async_to_json(
- await goto_child(node.archive_root, 'definitions')
- ),
- [v for v in m_def.split('/') if v not in ('', '#', 'definitions')],
- await goto_child(node.archive_root, ['metadata', 'upload_id']),
- await goto_child(node.archive_root, ['metadata', 'entry_id']),
- )
- # todo: !!!need to unify different formats!!!
- # check if m_def matches the pattern 'entry_id:example_id.example_section.example_quantity'
- regex = re.compile(r'entry_id:(.+)(?:\.(.+))+')
- if match := regex.match(m_def):
- entry_id = match.groups()[0]
- upload_id = Entry.objects(entry_id=entry_id).first().upload_id
- archive = self.load_archive(upload_id, entry_id)
- return __resolve_definition_in_archive(
- await async_to_json(await goto_child(archive, 'definitions')),
- list(match.groups()[1:]),
- upload_id,
- entry_id,
- )
-
- # further consider when only m_def_id is given, etc.
-
- # this is not likely to be reached
- # it does not work anyway
- proxy = SectionReference().normalize(m_def)
- proxy.m_proxy_context = context
- return proxy.section_cls.m_def
-
@classmethod
def validate_config(cls, key: str, config: RequestConfig):
if config.pagination is not None:
@@ -2685,7 +2757,7 @@ class ArchiveReader(GeneralReader):
return reader.sync_read(archive)
-class DefinitionReader(GeneralReader):
+class DefinitionReader(ArchiveLikeReader):
async def read(self, archive: Definition) -> dict:
response: dict = {Token.DEF: {}}
@@ -2814,17 +2886,10 @@ class DefinitionReader(GeneralReader):
# should never reach here
raise # noqa: PLE0704
- def __unwrap_subsection(__archive):
- return (
- __archive.sub_section.m_resolved()
- if isinstance(__archive, SubSection)
- else __archive
- )
-
if isinstance(value, RequestConfig):
# this is a leaf, resolve it according to the config
async def __resolve(__path, __target):
- __archive = __unwrap_subsection(__target)
+ __archive = _unwrap_subsection(__target)
if __archive is node.archive:
return
await self._resolve(
@@ -2846,7 +2911,7 @@ class DefinitionReader(GeneralReader):
elif isinstance(value, dict):
# this is a nested query, keep walking down the tree
async def __walk(__path, __target):
- __archive = __unwrap_subsection(__target)
+ __archive = _unwrap_subsection(__target)
if __archive is node.archive:
return
await self._walk(
@@ -2891,14 +2956,14 @@ class DefinitionReader(GeneralReader):
else ref_type.target_section_def
)
+ def __convert(m_def):
+ return _convert_ref_to_path_string(m_def.strict_reference())
+
def __override_path(q, s, v, p):
"""
Normalise all definition identifiers with unique global reference.
"""
- def __convert(m_def):
- return _convert_ref_to_path_string(m_def.strict_reference())
-
if isinstance(s, Quantity) and isinstance(v, dict):
if isinstance(s.type, Reference):
v['type_data'] = __convert(__unwrap_ref(s.type))
@@ -2917,6 +2982,36 @@ class DefinitionReader(GeneralReader):
return v
+ def __unique_name(item):
+ # quantities may have identical names in different sections
+ # cannot just use the name as the key
+ # sections are guaranteed to have unique names
+ return (
+ f'{item.m_parent.name}.{item.name}'
+ if isinstance(item, Quantity)
+ else item.name
+ )
+
+ #
+ # the actual logic starts here
+ #
+
+ # the following forces definitions being resolved per package
+ if config.export_whole_package:
+ pkg = node.archive
+ while not isinstance(pkg, Package) and pkg.m_parent is not None:
+ pkg = pkg.m_parent
+ if pkg is not node.archive:
+ node = await self._switch_root(
+ node.replace(archive=pkg),
+ inplace=config.resolve_inplace,
+ )
+
+ # use current path as the unique package identifier
+ # instead of generating a new one from definition
+ if not config.if_include('/'.join(node.current_path)):
+ return
+
# rewrite quantity type data with global reference
if not self._check_cache(node.current_path, config.hash):
self._cache_hash(node.current_path, config.hash)
@@ -2924,7 +3019,31 @@ class DefinitionReader(GeneralReader):
node.result_root,
node.current_path,
node.archive.m_to_dict(with_out_meta=True, transform=__override_path),
+ # the target location may contain a reference string already
+ # the string was added during switching root
+ # we allow it to be overwritten here
+ overwrite_existing_str=True,
)
+ if isinstance(node.archive, Package):
+ # always export the following for packages
+ for name in ('all_quantities', 'all_sub_sections', 'all_base_sections'):
+ container: set = set()
+ for section in node.archive.section_definitions:
+ target = getattr(section, name)
+ container.update(
+ _unwrap_subsection(v)
+ for v in (
+ target
+ if isinstance(target, (list, set))
+ else target.values()
+ )
+ )
+ output: dict = {__unique_name(v): __convert(v) for v in container}
+ await _populate_result(
+ node.result_root,
+ node.current_path + [name],
+ {k: v for k, v in sorted(output.items(), key=lambda x: x[1])},
+ )
if isinstance(node.archive, Quantity):
if isinstance(ref := node.archive.type, Reference):
@@ -2950,7 +3069,7 @@ class DefinitionReader(GeneralReader):
return
#
- # the following is for section
+ # the following is for section and package
#
# no need to recursively resolve all relevant definitions if the directive is plain
@@ -2966,32 +3085,37 @@ class DefinitionReader(GeneralReader):
):
return
- for name, unwrap in (
- ('extending_sections', False),
- ('base_sections', False),
- ('sub_sections', True),
- ('quantities', False),
- ):
- for index, base in enumerate(getattr(node.archive, name, [])):
- section = base.sub_section.m_resolved() if unwrap else base
- ref_str = section.strict_reference()
- path_stack = _convert_ref_to_path(ref_str)
- if section is node.archive or self._check_cache(
- path_stack, config.hash
- ):
- continue
- await self._resolve(
- await self._switch_root(
- node.replace(
- archive=section,
- current_path=node.current_path + [name, str(index)],
- visited_path=node.visited_path.union({ref_str}),
- current_depth=node.current_depth + 1,
+ async def __visit(_definition, _items):
+ for _name in _items:
+ for _index, _base in enumerate(getattr(_definition, _name, [])):
+ _section = _unwrap_subsection(_base)
+ _ref_str = _section.strict_reference()
+ _path_stack = _convert_ref_to_path(_ref_str)
+ if _section is _definition or self._check_cache(
+ _path_stack, config.hash
+ ):
+ continue
+ await self._resolve(
+ await self._switch_root(
+ node.replace(
+ archive=_section,
+ current_path=node.current_path + [_name, str(_index)],
+ visited_path=node.visited_path.union({_ref_str}),
+ current_depth=node.current_depth + 1,
+ ),
+ inplace=config.resolve_inplace,
),
- inplace=config.resolve_inplace,
- ),
- config,
- )
+ config,
+ )
+
+ if isinstance(node.archive, Package):
+ for section in node.archive.section_definitions:
+ await __visit(section, ('extending_sections', 'base_sections'))
+ else:
+ await __visit(
+ node.archive,
+ ('extending_sections', 'base_sections', 'sub_sections', 'quantities'),
+ )
@staticmethod
async def _switch_root(node: GraphNode, *, inplace: bool) -> GraphNode:
@@ -3002,6 +3126,27 @@ class DefinitionReader(GeneralReader):
if inplace:
return node
+ if isinstance(node.archive, Package):
+ return node.replace(
+ result_root=node.ref_result_root,
+ current_path=[
+ Token.UPLOADS,
+ node.archive.upload_id,
+ Token.ENTRIES,
+ node.archive.entry_id,
+ Token.ARCHIVE,
+ 'definitions',
+ ] # reconstruct the path to the definition in the archive
+ if node.archive.entry_id and node.archive.upload_id
+ else [
+ Token.METAINFO,
+ node.archive.name,
+ ], # otherwise a built-in package
+ )
+
+ # we always put a reference string at the current location
+ # since the section may belong to another package
+ # its definition may be placed in at the current location, or another location
ref_str: str = node.archive.strict_reference()
if not isinstance(node.archive, Quantity):
await _populate_result(
@@ -3019,6 +3164,158 @@ class DefinitionReader(GeneralReader):
return config
+class MetainfoBrowser(DefinitionReader):
+ """
+ A special implementation of definition reader.
+ """
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self._pagination_response: dict | None = None
+
+ def _apply_query(self, config: RequestConfig) -> list[str]:
+ if config.query is None:
+ all_keys: list = list(Package.registry.keys())
+ else:
+ raise NotImplementedError
+ # todo: implement query based filtering
+
+ total: int = len(all_keys)
+
+ default_pagination = config.pagination
+ if default_pagination is not None:
+ assert isinstance(default_pagination, MetainfoPagination)
+ all_keys = default_pagination.order_result(all_keys)
+ all_keys = default_pagination.paginate_result(all_keys, None)
+ else:
+ default_pagination = MetainfoPagination()
+
+ # we use the class member to cache the response
+ # it will be written to the result tree later
+ # we do not direct perform writing here to avoid turning all methods async
+ self._pagination_response = default_pagination.dict()
+ self._pagination_response['total'] = total
+
+ return all_keys
+
+ def _filter_registry(self, config: RequestConfig, omit_keys=None) -> Iterator[str]:
+ """
+ Filter the registry based on the given config.
+ """
+ for pkg_name in self._apply_query(config):
+ if not config.if_include(pkg_name):
+ continue
+ if omit_keys is not None and pkg_name in omit_keys:
+ continue
+ yield pkg_name
+
+ async def _generate_package(
+ self,
+ ) -> AsyncIterator[tuple[str, Package, RequestConfig | dict]]:
+ if isinstance(self.required_query, RequestConfig):
+ for name in self._filter_registry(self.required_query):
+ yield name, Package.registry[name], self.required_query
+ else:
+ has_wildcard: bool = GeneralReader.__WILDCARD__ in self.required_query
+
+ if GeneralReader.__CONFIG__ in self.required_query:
+ current_config: RequestConfig = self.required_query[
+ GeneralReader.__CONFIG__
+ ]
+ if has_wildcard:
+ child_config = self.required_query[GeneralReader.__WILDCARD__]
+ else:
+ child_config = current_config.new(
+ {
+ 'index': None, # ignore index requirements for children
+ 'query': None, # ignore query for children
+ 'pagination': None, # ignore pagination for children
+ },
+ retain_pattern=True, # alert: should the include/exclude pattern be retained?
+ )
+ for name in self._filter_registry(
+ current_config, omit_keys=self.required_query.keys()
+ ):
+ yield name, Package.registry[name], child_config
+ elif has_wildcard:
+ raise ValueError(
+ 'Wildcard is not supported when no parent config is defined.'
+ )
+
+ for key, value in self.required_query.items():
+ if key in (GeneralReader.__CONFIG__, GeneralReader.__WILDCARD__):
+ continue
+
+ if key in Package.registry:
+ yield key, Package.registry[key], value
+ elif key.startswith('entry_id:'):
+ try:
+ yield key, await self._retrieve_definition(key), value
+ except Exception as e:
+ self._log(f'Failed to retrieve definition: {e}')
+
+ async def read(self) -> dict: # type: ignore # noqa
+ response: dict = {}
+
+ if self.global_root is None:
+ self.global_root = response
+ has_global_root: bool = False
+ else:
+ has_global_root = True
+
+ current_config = self.global_config
+ if isinstance(self.required_query, dict):
+ current_config = self.required_query.get(
+ GeneralReader.__CONFIG__, current_config
+ )
+
+ async for pkg_name, pkg_definition, pkg_query in self._generate_package():
+ response.setdefault(pkg_name, {})
+ await self._walk(
+ GraphNode(
+ upload_id='__NONE__',
+ entry_id='__NONE__',
+ current_path=[pkg_name],
+ result_root=response,
+ ref_result_root=self.global_root,
+ archive=pkg_definition,
+ archive_root=None,
+ definition=None,
+ visited_path=set(),
+ current_depth=0,
+ reader=self,
+ ),
+ pkg_query,
+ current_config,
+ )
+
+ self._populate_error_list(response)
+
+ if not has_global_root:
+ self.global_root = None
+
+ if self._pagination_response is not None:
+ await _populate_result(
+ response, [Token.RESPONSE, 'pagination'], self._pagination_response
+ )
+ # reset the cache to ensure re-entrance
+ self._pagination_response = None
+
+ return response
+
+ @classmethod
+ def validate_config(cls, key: str, config: RequestConfig):
+ try:
+ if config.query is not None:
+ config.query = MetainfoQuery.parse_obj(config.query)
+ if config.pagination is not None:
+ config.pagination = MetainfoPagination.parse_obj(config.pagination)
+ except Exception as e:
+ raise ConfigError(str(e))
+
+ return config
+
+
__M_SEARCHABLE__: dict = {
Token.SEARCH: ElasticSearchReader,
Token.METADATA: ElasticSearchReader,
diff --git a/nomad/graph/model.py b/nomad/graph/model.py
index d9ff44bf51b4a5605a0f38c47e8fd894b3455bc2..c726b45536bc65b6fcb0a3bc1da6ab4353d830e7 100644
--- a/nomad/graph/model.py
+++ b/nomad/graph/model.py
@@ -25,7 +25,7 @@ from typing import FrozenSet, Optional, Union
from pydantic import BaseModel, Field, Extra, ValidationError, validator
-from nomad.app.v1.models import MetadataPagination, Metadata
+from nomad.app.v1.models import MetadataPagination, Metadata, Pagination, Direction
from nomad.app.v1.routers.datasets import DatasetPagination
from nomad.app.v1.routers.uploads import (
UploadProcDataQuery,
@@ -65,6 +65,37 @@ class EntryQuery(BaseModel):
)
+class MetainfoQuery(BaseModel):
+ pass
+
+
+class MetainfoPagination(Pagination):
+ def order_result(self, result):
+ return list(sorted(result, reverse=self.order == Direction.desc))
+
+ def paginate_result(self, result, pick_value):
+ if self.page is not None:
+ start = (self.page - 1) * self.page_size
+ end = start + self.page_size
+ elif self.page_offset is not None:
+ start = self.page_offset
+ end = start + self.page_size
+ elif self.page_after_value is not None:
+ start = 0
+ for index, item in enumerate(result):
+ if item == self.page_after_value:
+ start = index + 1
+ break
+ end = start + self.page_size
+ else:
+ start, end = 0, self.page_size
+
+ total_size = len(result)
+ first, last = min(start, total_size), min(end, total_size)
+
+ return [] if first == last else result[first:last]
+
+
class DirectiveType(Enum):
plain = 'plain'
resolved = 'resolved'
@@ -187,6 +218,13 @@ class RequestConfig(BaseModel):
The resolved quantity/section will be placed in the same archive.
""",
)
+ export_whole_package: bool = Field(
+ False,
+ description="""
+ Set to `True` to always get the whole package.
+ Set to `False` to definitions per section basis.
+ """,
+ )
include_definition: DefinitionType = Field(
DefinitionType.none,
description="""
@@ -220,6 +258,7 @@ class RequestConfig(BaseModel):
UploadProcDataPagination,
MetadataPagination,
EntryProcDataPagination,
+ MetainfoPagination,
] = Field(
None,
description="""
@@ -229,7 +268,9 @@ class RequestConfig(BaseModel):
Please refer to `DatasetPagination`, `UploadProcDataPagination`, `MetadataPagination` for details.
""",
)
- query: Union[dict, DatasetQuery, UploadProcDataQuery, Metadata, EntryQuery] = Field(
+ query: Union[
+ dict, DatasetQuery, UploadProcDataQuery, Metadata, EntryQuery, MetainfoQuery
+ ] = Field(
None,
description="""
The query configuration used for either mongo or elastic search.
diff --git a/nomad/metainfo/metainfo.py b/nomad/metainfo/metainfo.py
index 3e8305e6df34c4173b45b9e54d00d2b5349b884e..3574c029217cabc94e8fe9c44bc47be990c2df77 100644
--- a/nomad/metainfo/metainfo.py
+++ b/nomad/metainfo/metainfo.py
@@ -4075,6 +4075,9 @@ class Package(Definition):
return super().qualified_name()
def m_resolve_path(self, path_stack: list):
+ if len(path_stack) == 0:
+ return self
+
path_str = '/'.join(path_stack)
current_pos = path_stack.pop(0)
section = self.all_definitions.get(current_pos, None)
diff --git a/tests/graph/test_definition_reader.py b/tests/graph/test_definition_reader.py
index c7ece5f3091de5d23e31a970f4932399155ceabc..f67385897f1c1994fd2602742007eb02aff75e4d 100644
--- a/tests/graph/test_definition_reader.py
+++ b/tests/graph/test_definition_reader.py
@@ -139,6 +139,16 @@ def assert_dict(d1, d2):
},
id='plain-retrieval',
),
+ pytest.param(
+ {
+ 'm_request': {
+ 'directive': 'plain',
+ 'exclude': ['*test_definition_reader*'],
+ }
+ },
+ {'m_def': f'{prefix}/3'},
+ id='plain-retrieval-exclude',
+ ),
# now resolve all referenced quantities and sections
pytest.param(
{'m_request': {'directive': 'resolved'}},
@@ -584,7 +594,7 @@ def assert_dict(d1, d2):
),
],
)
-def test_definition_reader(query, result):
+def test_definition_reader(query: dict, result: dict):
with DefinitionReader(query) as reader:
response = remove_cache(reader.sync_read(m_def))
- assert_dict(response, result)
+ assert_dict(response, result)
diff --git a/tests/graph/test_graph_reader.py b/tests/graph/test_graph_reader.py
index 2f254db37a27e09d7b2c25ceb356229dcc5514ee..091c65c344b637cce6051abadb0bb823dd4a4c2c 100644
--- a/tests/graph/test_graph_reader.py
+++ b/tests/graph/test_graph_reader.py
@@ -2511,6 +2511,505 @@ def test_general_reader(json_dict, example_data_with_reference, user1):
)
+# noinspection DuplicatedCode,SpellCheckingInspection
+def test_metainfo_reader(mongo_infra, user1):
+ def increment():
+ n = 0
+ while True:
+ n += 1
+ yield n
+
+ counter = increment()
+
+ def __ge_print(msg, required, *, to_file: bool = False, result: dict = None):
+ with MongoReader(required, user=user1) as reader:
+ if result:
+ assert_dict(reader.sync_read(), result)
+ else:
+ rprint(f'\n\nExample: {next(counter)} -> {msg}:')
+ rprint(required)
+ if not to_file:
+ rprint('output:')
+ rprint(reader.sync_read())
+ else:
+ with open('archive_reader_test.json', 'w') as f:
+ f.write(json.dumps(reader.sync_read()))
+
+ __ge_print(
+ 'general start from metainfo',
+ {
+ Token.METAINFO: {
+ 'nomad.datamodel.metainfo.simulation.run': {
+ 'section_definitions[2]': {
+ 'm_request': {'directive': 'plain'},
+ }
+ }
+ }
+ },
+ result={
+ 'metainfo': {
+ 'nomad.datamodel.metainfo.simulation.run': {
+ 'section_definitions': [
+ None,
+ None,
+ {
+ 'name': 'MessageRun',
+ 'description': 'Contains warning, error, and info messages of the run.',
+ 'quantities': [
+ {
+ 'name': 'type',
+ 'description': 'Type of the message. Can be one of warning, error, info, debug.',
+ 'type': {'type_kind': 'python', 'type_data': 'str'},
+ 'shape': [],
+ },
+ {
+ 'name': 'value',
+ 'description': 'Value of the message of the computational program, given by type.',
+ 'type': {'type_kind': 'python', 'type_data': 'str'},
+ 'shape': [],
+ },
+ ],
+ },
+ ]
+ }
+ },
+ },
+ )
+
+ __ge_print(
+ 'general start from metainfo',
+ {
+ Token.METAINFO: {
+ 'm_request': {
+ 'include': ['*nomad.datamodel.metainfo.simulation.run'],
+ 'pagination': {'page_size': 50},
+ },
+ '*': {'m_request': {'index': [2]}},
+ }
+ },
+ result={
+ 'metainfo': {
+ 'nomad.datamodel.metainfo.simulation.run': {
+ 'name': 'nomad.datamodel.metainfo.simulation.run',
+ 'section_definitions': [
+ {
+ 'name': 'Program',
+ 'description': 'Contains the specifications of the program.',
+ 'quantities': [
+ {
+ 'name': 'name',
+ 'description': 'Specifies the name of the program that generated the data.',
+ 'categories': [
+ '/packages/0/category_definitions/0',
+ '/packages/0/category_definitions/1',
+ ],
+ 'type': {'type_kind': 'python', 'type_data': 'str'},
+ 'shape': [],
+ },
+ {
+ 'name': 'version',
+ 'description': 'Specifies the official release version of the program that was used.',
+ 'categories': [
+ '/packages/0/category_definitions/0',
+ '/packages/0/category_definitions/1',
+ ],
+ 'type': {'type_kind': 'python', 'type_data': 'str'},
+ 'shape': [],
+ },
+ {
+ 'name': 'version_internal',
+ 'description': 'Specifies a program version tag used internally for development purposes.\nAny kind of tagging system is supported, including git commit hashes.',
+ 'categories': [
+ '/packages/0/category_definitions/1'
+ ],
+ 'type': {'type_kind': 'python', 'type_data': 'str'},
+ },
+ {
+ 'name': 'compilation_datetime',
+ 'description': 'Contains the program compilation date and time from *Unix epoch* (00:00:00 UTC on\n1 January 1970) in seconds. For date and times without a timezone, the default\ntimezone GMT is used.',
+ 'categories': [
+ '/packages/0/category_definitions/0',
+ '/packages/0/category_definitions/1',
+ ],
+ 'type': {
+ 'type_kind': 'numpy',
+ 'type_data': 'float64',
+ },
+ 'shape': [],
+ 'unit': 'second',
+ },
+ {
+ 'name': 'compilation_host',
+ 'description': 'Specifies the host on which the program was compiled.',
+ 'categories': [
+ '/packages/0/category_definitions/0',
+ '/packages/0/category_definitions/1',
+ ],
+ 'type': {'type_kind': 'python', 'type_data': 'str'},
+ 'shape': [],
+ },
+ ],
+ },
+ {
+ 'name': 'TimeRun',
+ 'description': 'Contains information on timing information of the run.',
+ 'quantities': [
+ {
+ 'name': 'date_end',
+ 'description': 'Stores the end date of the run as time since the *Unix epoch* (00:00:00 UTC on 1\nJanuary 1970) in seconds. For date and times without a timezone, the default\ntimezone GMT is used.',
+ 'type': {
+ 'type_kind': 'numpy',
+ 'type_data': 'float64',
+ },
+ 'shape': [],
+ 'unit': 'second',
+ },
+ {
+ 'name': 'date_start',
+ 'description': 'Stores the start date of the run as time since the *Unix epoch* (00:00:00 UTC on 1\nJanuary 1970) in seconds. For date and times without a timezone, the default\ntimezone GMT is used.',
+ 'type': {
+ 'type_kind': 'numpy',
+ 'type_data': 'float64',
+ },
+ 'shape': [],
+ 'unit': 'second',
+ },
+ {
+ 'name': 'cpu1_end',
+ 'description': 'Stores the end time of the run on CPU 1.',
+ 'type': {
+ 'type_kind': 'numpy',
+ 'type_data': 'float64',
+ },
+ 'shape': [],
+ 'unit': 'second',
+ },
+ {
+ 'name': 'cpu1_start',
+ 'description': 'Stores the start time of the run on CPU 1.',
+ 'type': {
+ 'type_kind': 'numpy',
+ 'type_data': 'float64',
+ },
+ 'shape': [],
+ 'unit': 'second',
+ },
+ {
+ 'name': 'wall_end',
+ 'description': 'Stores the internal wall-clock time at the end of the run.',
+ 'type': {
+ 'type_kind': 'numpy',
+ 'type_data': 'float64',
+ },
+ 'shape': [],
+ 'unit': 'second',
+ },
+ {
+ 'name': 'wall_start',
+ 'description': 'Stores the internal wall-clock time from the start of the run.',
+ 'type': {
+ 'type_kind': 'numpy',
+ 'type_data': 'float64',
+ },
+ 'shape': [],
+ 'unit': 'second',
+ },
+ ],
+ },
+ {
+ 'name': 'MessageRun',
+ 'description': 'Contains warning, error, and info messages of the run.',
+ 'quantities': [
+ {
+ 'name': 'type',
+ 'description': 'Type of the message. Can be one of warning, error, info, debug.',
+ 'type': {'type_kind': 'python', 'type_data': 'str'},
+ 'shape': [],
+ },
+ {
+ 'name': 'value',
+ 'description': 'Value of the message of the computational program, given by type.',
+ 'type': {'type_kind': 'python', 'type_data': 'str'},
+ 'shape': [],
+ },
+ ],
+ },
+ {
+ 'name': 'Run',
+ 'description': 'Every section run represents a single call of a program.',
+ 'base_sections': [
+ 'metainfo/nomad.datamodel.data/section_definitions/0'
+ ],
+ 'quantities': [
+ {
+ 'name': 'calculation_file_uri',
+ 'description': 'Contains the nomad uri of a raw the data file connected to the current run. There\nshould be an value for the main_file_uri and all ancillary files.',
+ 'type': {'type_kind': 'python', 'type_data': 'str'},
+ 'shape': [],
+ },
+ {
+ 'name': 'clean_end',
+ 'description': 'Indicates whether this run terminated properly (true), or if it was killed or\nexited with an error code unequal to zero (false).',
+ 'type': {
+ 'type_kind': 'python',
+ 'type_data': 'bool',
+ },
+ 'shape': [],
+ },
+ {
+ 'name': 'raw_id',
+ 'description': 'An optional calculation id, if one is found in the code input/output files.',
+ 'type': {'type_kind': 'python', 'type_data': 'str'},
+ 'shape': [],
+ },
+ {
+ 'name': 'starting_run_ref',
+ 'description': 'Links the current section run to a section run containing the calculations from\nwhich the current section starts.',
+ 'categories': ['/category_definitions/0'],
+ 'type': {
+ 'type_kind': 'reference',
+ 'type_data': 'metainfo/nomad.datamodel.metainfo.simulation.run/section_definitions/3',
+ },
+ 'shape': [],
+ },
+ {
+ 'name': 'n_references',
+ 'description': 'Number of references to the current section calculation.',
+ 'type': {
+ 'type_kind': 'numpy',
+ 'type_data': 'int32',
+ },
+ 'shape': [],
+ },
+ {
+ 'name': 'runs_ref',
+ 'description': 'Links the the current section to other run sections. Such a link is necessary for\nexample for workflows that may contain a series of runs.',
+ 'categories': ['/category_definitions/0'],
+ 'type': {
+ 'type_kind': 'reference',
+ 'type_data': 'metainfo/nomad.datamodel.metainfo.simulation.run/section_definitions/3',
+ },
+ 'shape': ['n_references'],
+ },
+ ],
+ 'sub_sections': [
+ {
+ 'name': 'program',
+ 'sub_section': 'metainfo/nomad.datamodel.metainfo.simulation.run/section_definitions/0',
+ },
+ {
+ 'name': 'time_run',
+ 'sub_section': 'metainfo/nomad.datamodel.metainfo.simulation.run/section_definitions/1',
+ },
+ {
+ 'name': 'message',
+ 'sub_section': 'metainfo/nomad.datamodel.metainfo.simulation.run/section_definitions/2',
+ },
+ {
+ 'name': 'method',
+ 'sub_section': 'metainfo/nomad.datamodel.metainfo.simulation.method/section_definitions/44',
+ 'repeats': True,
+ },
+ {
+ 'name': 'system',
+ 'sub_section': 'metainfo/nomad.datamodel.metainfo.simulation.system/section_definitions/8',
+ 'repeats': True,
+ },
+ {
+ 'name': 'calculation',
+ 'sub_section': 'metainfo/nomad.datamodel.metainfo.simulation.calculation/section_definitions/36',
+ 'repeats': True,
+ },
+ ],
+ },
+ ],
+ 'category_definitions': [
+ {
+ 'name': 'AccessoryInfo',
+ 'description': 'Information that *in theory* should not affect the results of the calculations (e.g.,\ntiming).',
+ },
+ {
+ 'name': 'ProgramInfo',
+ 'description': 'Contains information on the program that generated the data, i.e. the program_name,\nprogram_version, program_compilation_host and program_compilation_datetime as direct\nchildren of this field.',
+ 'categories': ['/packages/0/category_definitions/0'],
+ },
+ ],
+ 'all_base_sections': {
+ 'ArchiveSection': 'metainfo/nomad.datamodel.data/section_definitions/0'
+ },
+ 'all_quantities': {
+ 'MessageRun.value': 'metainfo/nomad.datamodel.metainfo.simulation.run/section_definitions/2/quantities/1',
+ 'Run.calculation_file_uri': 'metainfo/nomad.datamodel.metainfo.simulation.run/section_definitions/3/quantities/0',
+ 'TimeRun.date_start': 'metainfo/nomad.datamodel.metainfo.simulation.run/section_definitions/1/quantities/1',
+ 'Run.n_references': 'metainfo/nomad.datamodel.metainfo.simulation.run/section_definitions/3/quantities/4',
+ 'Program.name': 'metainfo/nomad.datamodel.metainfo.simulation.run/section_definitions/0/quantities/0',
+ 'TimeRun.date_end': 'metainfo/nomad.datamodel.metainfo.simulation.run/section_definitions/1/quantities/0',
+ 'Run.starting_run_ref': 'metainfo/nomad.datamodel.metainfo.simulation.run/section_definitions/3/quantities/3',
+ 'Run.raw_id': 'metainfo/nomad.datamodel.metainfo.simulation.run/section_definitions/3/quantities/2',
+ 'Program.compilation_host': 'metainfo/nomad.datamodel.metainfo.simulation.run/section_definitions/0/quantities/4',
+ 'TimeRun.wall_end': 'metainfo/nomad.datamodel.metainfo.simulation.run/section_definitions/1/quantities/4',
+ 'TimeRun.cpu1_start': 'metainfo/nomad.datamodel.metainfo.simulation.run/section_definitions/1/quantities/3',
+ 'TimeRun.cpu1_end': 'metainfo/nomad.datamodel.metainfo.simulation.run/section_definitions/1/quantities/2',
+ 'Run.clean_end': 'metainfo/nomad.datamodel.metainfo.simulation.run/section_definitions/3/quantities/1',
+ 'TimeRun.wall_start': 'metainfo/nomad.datamodel.metainfo.simulation.run/section_definitions/1/quantities/5',
+ 'MessageRun.type': 'metainfo/nomad.datamodel.metainfo.simulation.run/section_definitions/2/quantities/0',
+ 'Program.version': 'metainfo/nomad.datamodel.metainfo.simulation.run/section_definitions/0/quantities/1',
+ 'Program.compilation_datetime': 'metainfo/nomad.datamodel.metainfo.simulation.run/section_definitions/0/quantities/3',
+ 'Program.version_internal': 'metainfo/nomad.datamodel.metainfo.simulation.run/section_definitions/0/quantities/2',
+ 'Run.runs_ref': 'metainfo/nomad.datamodel.metainfo.simulation.run/section_definitions/3/quantities/5',
+ },
+ 'all_sub_sections': {
+ 'MessageRun': 'metainfo/nomad.datamodel.metainfo.simulation.run/section_definitions/2',
+ 'TimeRun': 'metainfo/nomad.datamodel.metainfo.simulation.run/section_definitions/1',
+ 'System': 'metainfo/nomad.datamodel.metainfo.simulation.system/section_definitions/8',
+ 'Method': 'metainfo/nomad.datamodel.metainfo.simulation.method/section_definitions/44',
+ 'Calculation': 'metainfo/nomad.datamodel.metainfo.simulation.calculation/section_definitions/36',
+ 'Program': 'metainfo/nomad.datamodel.metainfo.simulation.run/section_definitions/0',
+ },
+ },
+ },
+ },
+ )
+
+ __ge_print(
+ 'general start from metainfo',
+ {
+ Token.METAINFO: {
+ 'm_request': {
+ 'include': ['*test_data'],
+ 'pagination': {'page_size': 50},
+ }
+ }
+ },
+ result={
+ 'metainfo': {
+ 'tests.processing.test_data': {
+ 'name': 'tests.processing.test_data',
+ 'section_definitions': [
+ {
+ 'name': 'TestBatchSample',
+ 'base_sections': [
+ 'metainfo/nomad.datamodel.data/section_definitions/1'
+ ],
+ 'quantities': [
+ {
+ 'name': 'batch_id',
+ 'description': 'Id for the batch',
+ 'type': {'type_kind': 'python', 'type_data': 'str'},
+ },
+ {
+ 'name': 'sample_number',
+ 'description': 'Sample index',
+ 'type': {'type_kind': 'python', 'type_data': 'int'},
+ },
+ {
+ 'm_annotations': {
+ 'eln': [{'component': 'RichTextEditQuantity'}]
+ },
+ 'name': 'comments',
+ 'description': 'Comments',
+ 'type': {'type_kind': 'python', 'type_data': 'str'},
+ },
+ ],
+ },
+ {
+ 'name': 'TestBatch',
+ 'base_sections': [
+ 'metainfo/nomad.datamodel.data/section_definitions/1'
+ ],
+ 'quantities': [
+ {
+ 'm_annotations': {
+ 'eln': [{'component': 'StringEditQuantity'}]
+ },
+ 'name': 'batch_id',
+ 'description': 'Id for the batch',
+ 'type': {'type_kind': 'python', 'type_data': 'str'},
+ },
+ {
+ 'm_annotations': {
+ 'eln': [{'component': 'NumberEditQuantity'}]
+ },
+ 'name': 'n_samples',
+ 'description': 'Number of samples in batch',
+ 'type': {'type_kind': 'python', 'type_data': 'int'},
+ },
+ {
+ 'name': 'sample_refs',
+ 'more': {
+ 'descriptions': 'The samples in the batch.',
+ 'type_data': 'metainfo/tests.processing.test_data/section_definitions/0',
+ },
+ 'type': {
+ 'type_kind': 'reference',
+ 'type_data': 'metainfo/tests.processing.test_data/section_definitions/0',
+ },
+ 'shape': ['*'],
+ },
+ ],
+ },
+ {
+ 'name': 'TestSection',
+ 'base_sections': [
+ 'metainfo/nomad.datamodel.data/section_definitions/0'
+ ],
+ },
+ {
+ 'name': 'TestReferenceSection',
+ 'base_sections': [
+ 'metainfo/nomad.datamodel.data/section_definitions/0'
+ ],
+ 'quantities': [
+ {
+ 'name': 'reference',
+ 'type': {
+ 'type_kind': 'reference',
+ 'type_data': 'metainfo/tests.processing.test_data/section_definitions/2',
+ },
+ }
+ ],
+ },
+ {
+ 'name': 'TestData',
+ 'base_sections': [
+ 'metainfo/nomad.datamodel.data/section_definitions/1'
+ ],
+ 'sub_sections': [
+ {
+ 'name': 'test_section',
+ 'sub_section': 'metainfo/tests.processing.test_data/section_definitions/2',
+ 'repeats': True,
+ },
+ {
+ 'name': 'reference_section',
+ 'sub_section': 'metainfo/tests.processing.test_data/section_definitions/3',
+ },
+ ],
+ },
+ ],
+ 'all_base_sections': {
+ 'ArchiveSection': 'metainfo/nomad.datamodel.data/section_definitions/0',
+ 'EntryData': 'metainfo/nomad.datamodel.data/section_definitions/1',
+ },
+ 'all_quantities': {
+ 'TestBatch.batch_id': 'metainfo/tests.processing.test_data/section_definitions/1/quantities/0',
+ 'TestReferenceSection.reference': 'metainfo/tests.processing.test_data/section_definitions/3/quantities/0',
+ 'TestBatchSample.batch_id': 'metainfo/tests.processing.test_data/section_definitions/0/quantities/0',
+ 'TestBatch.sample_refs': 'metainfo/tests.processing.test_data/section_definitions/1/quantities/2',
+ 'TestBatchSample.sample_number': 'metainfo/tests.processing.test_data/section_definitions/0/quantities/1',
+ 'TestBatch.n_samples': 'metainfo/tests.processing.test_data/section_definitions/1/quantities/1',
+ 'TestBatchSample.comments': 'metainfo/tests.processing.test_data/section_definitions/0/quantities/2',
+ },
+ 'all_sub_sections': {
+ 'TestSection': 'metainfo/tests.processing.test_data/section_definitions/2',
+ 'TestReferenceSection': 'metainfo/tests.processing.test_data/section_definitions/3',
+ },
+ },
+ },
+ },
+ )
+
+
# noinspection DuplicatedCode,SpellCheckingInspection
def test_general_reader_search(json_dict, example_data_with_reference, user1):
def increment():
@@ -2700,6 +3199,7 @@ def test_custom_schema_archive_and_definition(user1, custom_data):
'm_def': {
'm_request': {
'directive': 'plain',
+ 'export_whole_package': True,
},
},
}
@@ -2719,6 +3219,327 @@ def test_custom_schema_archive_and_definition(user1, custom_data):
'process_status': 'SUCCESS',
'upload_id': 'id_custom',
'warnings': [],
+ 'uploads': {
+ 'id_custom': {
+ 'entries': {
+ 'id_example': {
+ 'archive': {
+ 'definitions': {
+ 'section_definitions': [
+ {
+ 'name': 'MySection',
+ 'base_sections': [
+ 'metainfo/nomad.datamodel.data/section_definitions/1'
+ ],
+ 'quantities': [
+ {
+ 'name': 'my_quantity',
+ 'type': {
+ 'type_kind': 'python',
+ 'type_data': 'str',
+ },
+ },
+ {
+ 'name': 'datetime_list',
+ 'type': {
+ 'type_kind': 'custom',
+ 'type_data': 'nomad.metainfo.data_type.Datetime',
+ },
+ 'shape': ['*'],
+ },
+ ],
+ }
+ ],
+ 'name': 'test_package_name',
+ 'all_base_sections': {
+ 'ArchiveSection': 'metainfo/nomad.datamodel.data/section_definitions/0',
+ 'EntryData': 'metainfo/nomad.datamodel.data/section_definitions/1',
+ },
+ 'all_quantities': {
+ 'MySection.my_quantity': 'uploads/id_custom/entries/id_example/archive/definitions/section_definitions/0/quantities/0',
+ 'MySection.datetime_list': 'uploads/id_custom/entries/id_example/archive/definitions/section_definitions/0/quantities/1',
+ },
+ 'all_sub_sections': {},
+ }
+ }
+ }
+ }
+ }
+ },
+ 'archive': {
+ 'data': {
+ 'datetime_list': [
+ '2022-04-01T00:00:00+00:00',
+ '2022-04-02T00:00:00+00:00',
+ ],
+ 'my_quantity': 'test_value',
+ 'm_def': {
+ 'm_def': 'uploads/id_custom/entries/id_example/archive/definitions/section_definitions/0'
+ },
+ }
+ },
+ },
+ )
+
+ __entry_print(
+ 'custom',
+ {
+ Token.ARCHIVE: {
+ 'data': {
+ 'm_def': {
+ 'm_request': {
+ 'directive': 'resolved',
+ 'export_whole_package': True,
+ 'depth': 1,
+ },
+ },
+ }
+ },
+ },
+ result={
+ 'uploads': {
+ 'id_custom': {
+ 'entries': {
+ 'id_example': {
+ 'archive': {
+ 'definitions': {
+ 'name': 'test_package_name',
+ 'section_definitions': [
+ {
+ 'name': 'MySection',
+ 'base_sections': [
+ 'metainfo/nomad.datamodel.data/section_definitions/1'
+ ],
+ 'quantities': [
+ {
+ 'name': 'my_quantity',
+ 'type': {
+ 'type_kind': 'python',
+ 'type_data': 'str',
+ },
+ },
+ {
+ 'name': 'datetime_list',
+ 'type': {
+ 'type_kind': 'custom',
+ 'type_data': 'nomad.metainfo.data_type.Datetime',
+ },
+ 'shape': ['*'],
+ },
+ ],
+ }
+ ],
+ 'all_quantities': {
+ 'MySection.my_quantity': 'uploads/id_custom/entries/id_example/archive/definitions/section_definitions/0/quantities/0',
+ 'MySection.datetime_list': 'uploads/id_custom/entries/id_example/archive/definitions/section_definitions/0/quantities/1',
+ },
+ 'all_sub_sections': {},
+ 'all_base_sections': {
+ 'ArchiveSection': 'metainfo/nomad.datamodel.data/section_definitions/0',
+ 'EntryData': 'metainfo/nomad.datamodel.data/section_definitions/1',
+ },
+ 'base_sections': [
+ 'metainfo/nomad.datamodel.data/section_definitions/1'
+ ],
+ }
+ }
+ }
+ }
+ }
+ },
+ 'metainfo': {
+ 'nomad.datamodel.data': {
+ 'name': 'nomad.datamodel.data',
+ 'section_definitions': [
+ {
+ 'name': 'ArchiveSection',
+ 'description': 'Base class for sections in a NOMAD archive. Provides a framework for custom section normalization via the `normalize` function.',
+ },
+ {
+ 'name': 'EntryData',
+ 'description': 'An empty base section definition. This can be used to add new top-level sections to an entry.',
+ 'base_sections': [
+ 'metainfo/nomad.datamodel.data/section_definitions/0'
+ ],
+ },
+ {
+ 'name': 'Author',
+ 'description': 'A person that is author of data in NOMAD or references by NOMAD.',
+ 'quantities': [
+ {
+ 'm_annotations': {
+ 'elasticsearch': [
+ 'viewers.name',
+ 'viewers.name.text',
+ 'viewers.name__suggestion',
+ ]
+ },
+ 'name': 'name',
+ 'type': {'type_kind': 'python', 'type_data': 'str'},
+ 'virtual': True,
+ },
+ {
+ 'name': 'first_name',
+ 'description': 'The users first name (including all other given names)',
+ 'type': {
+ 'type_kind': 'custom',
+ 'type_data': 'nomad.metainfo.data_type.Capitalized',
+ },
+ },
+ {
+ 'name': 'last_name',
+ 'description': 'The users last name',
+ 'type': {
+ 'type_kind': 'custom',
+ 'type_data': 'nomad.metainfo.data_type.Capitalized',
+ },
+ },
+ {
+ 'name': 'email',
+ 'type': {'type_kind': 'python', 'type_data': 'str'},
+ },
+ {
+ 'name': 'affiliation',
+ 'description': 'The name of the company and institutes the user identifies with',
+ 'type': {'type_kind': 'python', 'type_data': 'str'},
+ },
+ {
+ 'name': 'affiliation_address',
+ 'description': 'The address of the given affiliation',
+ 'type': {'type_kind': 'python', 'type_data': 'str'},
+ },
+ ],
+ },
+ {
+ 'm_annotations': {'pydantic': ['PydanticModel']},
+ 'name': 'User',
+ 'description': 'A NOMAD user. Typically a NOMAD user has a NOMAD account. The user related data is managed by\nNOMAD keycloak user-management system. Users are used to denote authors,\nreviewers, and owners of datasets.',
+ 'base_sections': [
+ 'metainfo/nomad.datamodel.data/section_definitions/2'
+ ],
+ 'quantities': [
+ {
+ 'm_annotations': {
+ 'elasticsearch': ['viewers.user_id']
+ },
+ 'name': 'user_id',
+ 'description': 'The unique, persistent keycloak UUID',
+ 'type': {'type_kind': 'python', 'type_data': 'str'},
+ },
+ {
+ 'name': 'username',
+ 'description': 'The unique, persistent, user chosen username',
+ 'type': {'type_kind': 'python', 'type_data': 'str'},
+ },
+ {
+ 'name': 'created',
+ 'description': 'The time the account was created',
+ 'type': {
+ 'type_kind': 'custom',
+ 'type_data': 'nomad.metainfo.data_type.Datetime',
+ },
+ },
+ {
+ 'name': 'repo_user_id',
+ 'description': 'Optional, legacy user id from the old NOMAD CoE repository.',
+ 'type': {'type_kind': 'python', 'type_data': 'str'},
+ },
+ {
+ 'name': 'is_admin',
+ 'description': 'Bool that indicated, iff the user the use admin user',
+ 'type': {
+ 'type_kind': 'python',
+ 'type_data': 'bool',
+ },
+ 'virtual': True,
+ },
+ {
+ 'name': 'is_oasis_admin',
+ 'type': {
+ 'type_kind': 'python',
+ 'type_data': 'bool',
+ },
+ 'default': False,
+ },
+ ],
+ },
+ ],
+ 'category_definitions': [
+ {'name': 'EntryDataCategory'},
+ {
+ 'name': 'ElnIntegrationCategory',
+ 'label': 'Third-party ELN Integration',
+ 'categories': ['/category_definitions/0'],
+ },
+ {
+ 'name': 'BasicElnCategory',
+ 'label': 'Basic ELN',
+ 'categories': ['/category_definitions/0'],
+ },
+ {
+ 'name': 'ElnExampleCategory',
+ 'label': 'Example ELNs',
+ 'categories': ['/category_definitions/0'],
+ },
+ {
+ 'name': 'UseCaseElnCategory',
+ 'label': 'Use-cases',
+ 'categories': ['/category_definitions/0'],
+ },
+ {
+ 'name': 'WorkflowsElnCategory',
+ 'label': 'Workflows',
+ 'categories': ['/category_definitions/0'],
+ },
+ ],
+ 'all_quantities': {
+ 'Author.name': 'metainfo/nomad.datamodel.data/section_definitions/2/quantities/0',
+ 'Author.first_name': 'metainfo/nomad.datamodel.data/section_definitions/2/quantities/1',
+ 'Author.last_name': 'metainfo/nomad.datamodel.data/section_definitions/2/quantities/2',
+ 'Author.email': 'metainfo/nomad.datamodel.data/section_definitions/2/quantities/3',
+ 'Author.affiliation': 'metainfo/nomad.datamodel.data/section_definitions/2/quantities/4',
+ 'Author.affiliation_address': 'metainfo/nomad.datamodel.data/section_definitions/2/quantities/5',
+ 'User.user_id': 'metainfo/nomad.datamodel.data/section_definitions/3/quantities/0',
+ 'User.username': 'metainfo/nomad.datamodel.data/section_definitions/3/quantities/1',
+ 'User.created': 'metainfo/nomad.datamodel.data/section_definitions/3/quantities/2',
+ 'User.repo_user_id': 'metainfo/nomad.datamodel.data/section_definitions/3/quantities/3',
+ 'User.is_admin': 'metainfo/nomad.datamodel.data/section_definitions/3/quantities/4',
+ 'User.is_oasis_admin': 'metainfo/nomad.datamodel.data/section_definitions/3/quantities/5',
+ },
+ 'all_sub_sections': {},
+ 'all_base_sections': {
+ 'ArchiveSection': 'metainfo/nomad.datamodel.data/section_definitions/0',
+ 'Author': 'metainfo/nomad.datamodel.data/section_definitions/2',
+ },
+ }
+ },
+ 'archive': {
+ 'data': {
+ 'm_def': {
+ 'm_def': 'uploads/id_custom/entries/id_example/archive/definitions/section_definitions/0'
+ }
+ }
+ },
+ },
+ )
+
+ __entry_print(
+ 'custom',
+ {
+ Token.ARCHIVE: {
+ 'data': {
+ 'm_request': {
+ 'directive': 'plain',
+ },
+ 'm_def': {
+ 'm_request': {
+ 'directive': 'plain',
+ },
+ },
+ }
+ },
+ },
+ result={
'uploads': {
'id_custom': {
'entries': {