diff --git a/nomad/app/v1/models/graph/graph_models.py b/nomad/app/v1/models/graph/graph_models.py
index f9eaac851f050b923dbe983b357fd86d555fa2b0..00eb8e2b3e2b94843ca41e8aaefd2f577d498c10 100644
--- a/nomad/app/v1/models/graph/graph_models.py
+++ b/nomad/app/v1/models/graph/graph_models.py
@@ -20,7 +20,7 @@ from __future__ import annotations
from typing import Optional, List, Union, Any, Literal
from pydantic import BaseModel, Field, Extra
-from nomad.metainfo import Package
+from nomad.groups import UserGroupModel
from nomad.graph.model import (
RequestConfig,
DatasetQuery,
@@ -240,6 +240,15 @@ class GraphMetainfo(BaseModel):
m_children: MSection
+class GraphGroup(mapped(UserGroupModel, owner=GraphUser, members=List[GraphUser])): # type: ignore
+ m_errors: List[Error]
+
+
+class GraphGroups(BaseModel):
+ m_errors: List[Error]
+ m_children: GraphGroup
+
+
class Graph(BaseModel):
users: GraphUsers
entries: GraphEntries
@@ -247,6 +256,7 @@ class Graph(BaseModel):
datasets: GraphDatasets
search: GraphSearch
metainfo: GraphMetainfo
+ groups: GraphGroups
GraphRequest = generate_request_model(Graph)
diff --git a/nomad/app/v1/routers/groups.py b/nomad/app/v1/routers/groups.py
index ae5361d00bf561f3057222ca5095abbb899310c6..e7a3f34aaed9b7b50fd82ffc03c11fbf2e3edc71 100644
--- a/nomad/app/v1/routers/groups.py
+++ b/nomad/app/v1/routers/groups.py
@@ -46,7 +46,7 @@ class UserGroupEdit(BaseModel):
description=group_name_description,
min_length=3,
max_length=32,
- regex=r'^[a-zA-Z0-9][a-zA-Z0-9 \.\_\-]+[a-zA-Z0-9]$',
+ regex=r'^[a-zA-Z0-9][a-zA-Z0-9 ._\-]+[a-zA-Z0-9]$',
)
members: Optional[Set[str]] = Field(
default=None, description=group_members_description
diff --git a/nomad/graph/graph_reader.py b/nomad/graph/graph_reader.py
index 29ded7015f26658106c778373e182217a1daa4b4..87d40ea078917841885da1288576e6eb056cf689 100644
--- a/nomad/graph/graph_reader.py
+++ b/nomad/graph/graph_reader.py
@@ -64,7 +64,10 @@ from nomad.graph.model import (
EntryQuery,
MetainfoQuery,
MetainfoPagination,
+ UserGroupQuery,
+ UserGroupPagination,
)
+from nomad.groups import UserGroup
from nomad.metainfo import (
SubSection,
QuantityReference,
@@ -101,6 +104,8 @@ class Token:
USERS = 'users'
DATASET = 'dataset'
DATASETS = 'm_datasets'
+ GROUP = 'group'
+ GROUPS = 'groups'
METAINFO = 'metainfo'
SEARCH = 'search'
ERROR = 'm_errors'
@@ -125,6 +130,14 @@ def dataset_to_pydantic(item):
return item.to_json()
+def group_to_pydantic(item):
+ """
+ Do NOT optimise this function.
+ Function names are used to determine the type of the object.
+ """
+ return item.to_json()
+
+
class ArchiveError(Exception):
"""
An exception raised when an error occurs in the archive.
@@ -571,6 +584,9 @@ def _normalise_required(
if name in GeneralReader.__USER_ID__ or name == Token.USER or name == Token.USERS:
reader_type = UserReader
+ elif name == Token.GROUP or name == Token.GROUPS:
+ reader_type = UserGroupReader
+ can_query = True
elif name in GeneralReader.__UPLOAD_ID__:
reader_type = UploadReader
elif name == Token.UPLOAD or name == Token.UPLOADS:
@@ -748,6 +764,8 @@ class GeneralReader:
'writers',
'entry_coauthors',
'user_id',
+ 'owner', # from UserGroup
+ 'members', # from UserGroup
}
# controls the names of fields that are treated as entry id, for those fields,
# implicit resolve is supported and explicit resolve does not require an explicit resolve type
@@ -894,6 +912,45 @@ class GeneralReader:
return user.m_to_dict(with_out_meta=True, include_derived=True)
+ async def _overwrite_group(self, item: UserGroup):
+ # todo: this is a quick dirty fix to convert the group object to a dictionary
+ # todo: it shall be formalised in the future
+ group_dict = item.to_mongo().to_dict()
+ group_dict['group_id'] = group_dict.pop('_id', None)
+
+ # to be consistent with the other parts
+ # all user ids are resolved to user objects
+ group_dict['owner'] = await self.retrieve_user(group_dict['owner'])
+ group_dict['members'] = [
+ await self.retrieve_user(member) for member in group_dict['members']
+ ]
+
+ return group_dict
+
+ async def retrieve_group(self, group_id: str) -> str | dict:
+ """
+ Retrieve the group for the given group id.
+ Returns a plain dictionary if the group is found, otherwise return the given group id.
+ """
+
+ def _retrieve():
+ return UserGroup.objects(group_id=group_id).first()
+
+ try:
+ group: UserGroup = await asyncio.to_thread(_retrieve)
+ except Exception as e:
+ self._log(str(e), to_response=False)
+ return group_id
+
+ if group is None:
+ self._log(
+ f'The value {group_id} is not a valid group id.',
+ error_type=QueryError.NOTFOUND,
+ )
+ return group_id
+
+ return await self._overwrite_group(group)
+
async def _overwrite_upload(self, item: Upload):
plain_dict = orjson.loads(upload_to_pydantic(item).json())
if n_entries := plain_dict.pop('entries', None):
@@ -1306,6 +1363,25 @@ class MongoReader(GeneralReader):
return config.query.dict(exclude_unset=True), self.datasets.filter(mongo_query)
+ async def _query_groups(self, config: RequestConfig):
+ # todo: extend and refine the query
+ if config.query:
+ assert isinstance(config.query, UserGroupQuery)
+ default_query = config.query
+ else:
+ default_query = UserGroupQuery(user_id=self.user.user_id)
+
+ # replace shortcut for myself
+ if default_query.user_id in ('me', None):
+ default_query.user_id = self.user.user_id
+
+ mongo_query = Q()
+ if default_query.user_id:
+ mongo_query &= Q(members=default_query.user_id)
+
+ # todo: maybe it is necessary to further refine the scope based on current user's visibility
+ return default_query.dict(exclude_unset=True), UserGroup.objects(mongo_query)
+
async def _normalise(
self, mongo_result, config: RequestConfig, transformer: Callable
) -> tuple[dict, PaginationResponse | None]:
@@ -1345,6 +1421,8 @@ class MongoReader(GeneralReader):
return _item.dataset_id
if transformer == entry_to_pydantic:
return _item.entry_id
+ if transformer == group_to_pydantic:
+ return _item.group_id
raise ValueError(f'Should not reach here.')
@@ -1370,6 +1448,11 @@ class MongoReader(GeneralReader):
v['entry_id']: v
for v in [self._overwrite_entry(item) for item in mongo_result]
}
+ elif transformer == group_to_pydantic:
+ mongo_dict = {
+ v['group_id']: v
+ for v in [await self._overwrite_group(item) for item in mongo_result]
+ }
else:
raise ValueError(f'Should not reach here.')
@@ -1655,6 +1738,9 @@ class MongoReader(GeneralReader):
if key == Token.DATASET or key == Token.DATASETS:
await offload_func(await self._query_datasets(config), dataset_to_pydantic)
return True
+ if key == Token.GROUP or key == Token.GROUPS:
+ await offload_func(await self._query_groups(config), group_to_pydantic)
+ return True
if key == Token.USER or key == Token.USERS:
await offload_func(
(
@@ -1988,42 +2074,50 @@ class UserReader(MongoReader):
if user_id == 'me':
user_id = self.user.user_id
- mongo_query = (
- Q(main_author=user_id) | Q(reviewers=user_id) | Q(coauthors=user_id)
- )
- # self.user must have access to the upload
- if user_id != self.user.user_id and not self.user.is_admin:
- mongo_query &= (
- Q(main_author=self.user.user_id)
- | Q(reviewers=self.user.user_id)
- | Q(coauthors=self.user.user_id)
+ if isinstance(target_user := await self.retrieve_user(user_id), str):
+ # does not exist
+ self._log(
+ f'User ID {user_id} does not exist.', error_type=QueryError.NOTFOUND
+ )
+ else:
+ mongo_query = (
+ Q(main_author=user_id) | Q(reviewers=user_id) | Q(coauthors=user_id)
)
+ # self.user must have access to the upload
+ if user_id != self.user.user_id and not self.user.is_admin:
+ mongo_query &= (
+ Q(main_author=self.user.user_id)
+ | Q(reviewers=self.user.user_id)
+ | Q(coauthors=self.user.user_id)
+ )
- self.uploads = Upload.objects(mongo_query)
- self.entries = Entry.objects(upload_id__in=[v.upload_id for v in self.uploads])
- self.datasets = Dataset.m_def.a_mongo.objects(
- dataset_id__in=set(
- v for e in self.entries if e.datasets for v in e.datasets
+ self.uploads = Upload.objects(mongo_query)
+ self.entries = Entry.objects(
+ upload_id__in=[v.upload_id for v in self.uploads]
+ )
+ self.datasets = Dataset.m_def.a_mongo.objects(
+ dataset_id__in=set(
+ v for e in self.entries if e.datasets for v in e.datasets
+ )
)
- )
- await self._walk(
- GraphNode(
- upload_id='__NOT_NEEDED__',
- entry_id='__NOT_NEEDED__',
- current_path=[],
- result_root=response,
- ref_result_root=self.global_root,
- archive=await self.retrieve_user(user_id),
- archive_root=None,
- definition=None,
- visited_path=set(),
- current_depth=0,
- reader=self,
- ),
- self.required_query,
- self.global_config,
- )
+ await self._walk(
+ GraphNode(
+ upload_id='__NOT_NEEDED__',
+ entry_id='__NOT_NEEDED__',
+ current_path=[],
+ result_root=response,
+ ref_result_root=self.global_root,
+ archive=target_user,
+ archive_root=None,
+ definition=None,
+ visited_path=set(),
+ current_depth=0,
+ reader=self,
+ ),
+ self.required_query,
+ self.global_config,
+ )
self._populate_error_list(response)
@@ -2039,7 +2133,62 @@ class UserReader(MongoReader):
if config.pagination is not None:
raise ConfigError('User reader does not support pagination.')
- return MongoReader.validate_config(key, config)
+ return super().validate_config(key, config)
+
+
+class UserGroupReader(MongoReader):
+ # noinspection PyMethodOverriding
+ async def read(self, group_id: str): # type: ignore
+ response: dict = {}
+
+ if self.global_root is None:
+ self.global_root = response
+ has_global_root: bool = False
+ else:
+ has_global_root = True
+
+ if isinstance(target_group := await self.retrieve_group(group_id), str):
+ # does not exist
+ self._log(
+ f'Group ID {group_id} does not exist.', error_type=QueryError.NOTFOUND
+ )
+ else:
+ await self._walk(
+ GraphNode(
+ upload_id='__NOT_NEEDED__',
+ entry_id='__NOT_NEEDED__',
+ current_path=[],
+ result_root=response,
+ ref_result_root=self.global_root,
+ archive=target_group,
+ archive_root=None,
+ definition=None,
+ visited_path=set(),
+ current_depth=0,
+ reader=self,
+ ),
+ self.required_query,
+ self.global_config,
+ )
+
+ self._populate_error_list(response)
+
+ if not has_global_root:
+ self.global_root = None
+
+ return response
+
+ @classmethod
+ def validate_config(cls, key: str, config: RequestConfig):
+ try:
+ if config.query is not None:
+ config.query = UserGroupQuery.parse_obj(config.query)
+ if config.pagination is not None:
+ config.pagination = UserGroupPagination.parse_obj(config.pagination)
+ except Exception as e:
+ raise ConfigError(str(e))
+
+ return super().validate_config(key, config)
class FileSystemReader(GeneralReader):
@@ -2403,7 +2552,7 @@ class ArchiveReader(ArchiveLikeReader):
if required.pop(GeneralReader.__WILDCARD__, None):
self._log(
- "Wildcard '*' as field name is not supported in archive query as its data is not homogeneous",
+ "Wildcard '*' as field name is not supported in archive query as its data is not homogeneous.",
error_type=QueryError.NOTFOUND,
)
@@ -3327,4 +3476,6 @@ __M_SEARCHABLE__: dict = {
Token.USERS: UserReader,
Token.DATASET: DatasetReader,
Token.DATASETS: DatasetReader,
+ Token.GROUP: UserGroupReader,
+ Token.GROUPS: UserGroupReader,
}
diff --git a/nomad/graph/model.py b/nomad/graph/model.py
index c726b45536bc65b6fcb0a3bc1da6ab4353d830e7..b4db9892e3ed3eaaf5512550a1da07b1bcf2d2e1 100644
--- a/nomad/graph/model.py
+++ b/nomad/graph/model.py
@@ -96,6 +96,23 @@ class MetainfoPagination(Pagination):
return [] if first == last else result[first:last]
+class UserGroupQuery(BaseModel):
+ # todo: define query specifics
+ user_id: str = Field(None)
+
+
+class UserGroupPagination(Pagination):
+ # todo: refine pagination logic
+ def order_result(self, result):
+ if self.order_by is None:
+ return result
+
+ prefix: str = '-' if self.order == Direction.desc else '+'
+ order_list: list = [f'{prefix}{self.order_by}', 'group_id']
+
+ return result.order_by(*order_list)
+
+
class DirectiveType(Enum):
plain = 'plain'
resolved = 'resolved'
@@ -253,12 +270,13 @@ class RequestConfig(BaseModel):
)
pagination: Union[
dict,
- RawDirPagination,
DatasetPagination,
- UploadProcDataPagination,
- MetadataPagination,
EntryProcDataPagination,
+ MetadataPagination,
MetainfoPagination,
+ RawDirPagination,
+ UploadProcDataPagination,
+ UserGroupPagination,
] = Field(
None,
description="""
@@ -269,7 +287,13 @@ class RequestConfig(BaseModel):
""",
)
query: Union[
- dict, DatasetQuery, UploadProcDataQuery, Metadata, EntryQuery, MetainfoQuery
+ dict,
+ DatasetQuery,
+ EntryQuery,
+ Metadata,
+ MetainfoQuery,
+ UploadProcDataQuery,
+ UserGroupQuery,
] = Field(
None,
description="""
@@ -278,7 +302,8 @@ class RequestConfig(BaseModel):
It can only be defined at the root levels including Token.ENTRIES, Token.UPLOADS and 'm_datasets'.
For Token.ENTRIES, the query is used in elastic search. It must comply with `WithQuery`.
For Token.UPLOADS, the query is used in mongo search. It must comply with `UploadProcDataQuery`.
- For 'm_datasets', the query is used in mongo search. It must comply with `DatasetQuery`.
+ For Token.DATASETS, the query is used in mongo search. It must comply with `DatasetQuery`.
+ For Token.GROUPS, the query is used in mongo search. It must comply with `UserGroupQuery`.
""",
)
diff --git a/nomad/groups.py b/nomad/groups.py
index c0637226a440dbdcf019410791ffd80f381db9af..d002d66ee81a240208ee45793c469888423b12cc 100644
--- a/nomad/groups.py
+++ b/nomad/groups.py
@@ -15,16 +15,31 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+
+from __future__ import annotations
+
import operator
from functools import reduce
from typing import Iterable, Optional, Union
from mongoengine import Document, ListField, StringField
from mongoengine.queryset.visitor import Q
+from pydantic import BaseModel, Field
from nomad.utils import create_uuid
+# todo: this is just my quick fix to get things glued together
+# todo: UserGroup shall be a MSection just like User so that
+# todo: mongo document and pydantic model can be generated from the same source
+# todo: see also dataset
+class UserGroupModel(BaseModel):
+ group_id: str = Field()
+ group_name: Optional[str] = Field()
+ owner: str = Field()
+ members: Optional[list[str]] = Field()
+
+
class UserGroup(Document):
"""
A group of users. One user is the owner, all others are members.
diff --git a/tests/graph/test_graph_reader.py b/tests/graph/test_graph_reader.py
index 091c65c344b637cce6051abadb0bb823dd4a4c2c..e62ab1e4581c59699dd65e5266cbc7b2a3d976e0 100644
--- a/tests/graph/test_graph_reader.py
+++ b/tests/graph/test_graph_reader.py
@@ -97,16 +97,18 @@ user_dict = {
}
-# noinspection SpellCheckingInspection,DuplicatedCode
-def test_remote_reference(json_dict, example_data_with_reference, user1):
- def increment():
- n = 0
- while True:
- n += 1
- yield n
+def increment():
+ n = 0
+ while True:
+ n += 1
+ yield n
+
- counter = increment()
+counter = increment()
+
+# noinspection SpellCheckingInspection,DuplicatedCode
+def test_remote_reference(json_dict, example_data_with_reference, user1):
def __user_print(msg, required, *, result: dict = None):
with UserReader(required, user=user1) as reader:
if result:
@@ -2117,15 +2119,103 @@ def test_remote_reference(json_dict, example_data_with_reference, user1):
# noinspection DuplicatedCode,SpellCheckingInspection
-def test_general_reader(json_dict, example_data_with_reference, user1):
- def increment():
- n = 0
- while True:
- n += 1
- yield n
+def test_group_reader(groups_function, user1):
+ def __ge_print(msg, required, *, to_file: bool = False, result: dict = None):
+ with MongoReader(required, user=user1) as reader:
+ if result:
+ assert_dict(reader.sync_read(), result)
+ else:
+ rprint(f'\n\nExample: {next(counter)} -> {msg}:')
+ rprint(required)
+ if not to_file:
+ rprint('output:')
+ rprint(reader.sync_read())
+ else:
+ with open('archive_reader_test.json', 'w') as f:
+ f.write(json.dumps(reader.sync_read()))
+
+ __ge_print(
+ 'general start from group',
+ {
+ Token.GROUP: {
+ 'GGGGGGGGGGGGGGGGGGGG14': '*',
+ }
+ },
+ result={
+ 'group': {
+ 'GGGGGGGGGGGGGGGGGGGG14': {
+ 'group_id': 'GGGGGGGGGGGGGGGGGGGG14',
+ 'group_name': 'Group 14',
+ 'owner': {
+ 'name': 'Sheldon Cooper',
+ 'first_name': 'Sheldon',
+ 'last_name': 'Cooper',
+ 'email': 'sheldon.cooper@nomad-coe.eu',
+ 'user_id': '00000000-0000-0000-0000-000000000001',
+ 'username': 'scooper',
+ 'is_admin': False,
+ 'is_oasis_admin': True,
+ },
+ 'members': [
+ {
+ 'name': 'Rajesh Koothrappali',
+ 'first_name': 'Rajesh',
+ 'last_name': 'Koothrappali',
+ 'email': 'rajesh.koothrappali@nomad-fairdi.tests.de',
+ 'user_id': '00000000-0000-0000-0000-000000000004',
+ 'username': 'rkoothrappali',
+ 'is_admin': False,
+ }
+ ],
+ }
+ }
+ },
+ )
+ __ge_print(
+ 'general start from group',
+ {
+ Token.GROUP: {
+ 'GGGGGGGGGGGGGGGGGGGG14': {
+ 'owner': {
+ 'email': '*',
+ },
+ },
+ }
+ },
+ result={
+ 'group': {
+ 'GGGGGGGGGGGGGGGGGGGG14': {
+ 'owner': {'email': 'sheldon.cooper@nomad-coe.eu'}
+ }
+ }
+ },
+ )
+ __ge_print(
+ 'general start from group with query',
+ {
+ Token.GROUP: {
+ 'm_request': {
+ 'query': {'user_id': '00000000-0000-0000-0000-000000000004'}
+ },
+ '*': {
+ 'owner': {
+ 'email': '*',
+ },
+ },
+ }
+ },
+ result={
+ 'group': {
+ 'GGGGGGGGGGGGGGGGGGGG14': {
+ 'owner': {'email': 'sheldon.cooper@nomad-coe.eu'}
+ },
+ }
+ },
+ )
- counter = increment()
+# noinspection DuplicatedCode,SpellCheckingInspection
+def test_general_reader(json_dict, example_data_with_reference, user1):
def __ge_print(msg, required, *, to_file: bool = False, result: dict = None):
with MongoReader(required, user=user1) as reader:
if result:
@@ -2513,14 +2603,6 @@ def test_general_reader(json_dict, example_data_with_reference, user1):
# noinspection DuplicatedCode,SpellCheckingInspection
def test_metainfo_reader(mongo_infra, user1):
- def increment():
- n = 0
- while True:
- n += 1
- yield n
-
- counter = increment()
-
def __ge_print(msg, required, *, to_file: bool = False, result: dict = None):
with MongoReader(required, user=user1) as reader:
if result:
@@ -3012,14 +3094,6 @@ def test_metainfo_reader(mongo_infra, user1):
# noinspection DuplicatedCode,SpellCheckingInspection
def test_general_reader_search(json_dict, example_data_with_reference, user1):
- def increment():
- n = 0
- while True:
- n += 1
- yield n
-
- counter = increment()
-
def __ge_print(msg, required, *, to_file: bool = False, result: dict = None):
with MongoReader(required, user=user1) as reader:
if result:
@@ -3162,14 +3236,6 @@ data:
def test_custom_schema_archive_and_definition(user1, custom_data):
- def increment():
- n = 0
- while True:
- n += 1
- yield n
-
- counter = increment()
-
def __entry_print(msg, required, *, to_file: bool = False, result: dict = None):
with EntryReader(required, user=user1) as reader:
response = reader.sync_read('id_example')