Commit c61e013d authored by David Sikter's avatar David Sikter
Browse files

Base functionality of new edit api

parent a3a0b024
......@@ -30,13 +30,14 @@ from pydantic import ( # pylint: disable=unused-import
validator,
root_validator,
)
from pydantic.main import create_model
import datetime
import numpy as np
import re
import fnmatch
import json
from nomad import datamodel # pylint: disable=unused-import
from nomad import datamodel, metainfo # pylint: disable=unused-import
from nomad.utils import strip
from nomad.metainfo import Datetime, MEnum
from nomad.metainfo.elasticsearch_extension import DocumentType, material_entry_type, material_type
......@@ -51,6 +52,44 @@ Value = Union[StrictInt, StrictFloat, StrictBool, str, datetime.datetime]
ComparableValue = Union[StrictInt, StrictFloat, str, datetime.datetime]
class Owner(str, enum.Enum):
'''
The `owner` allows to limit the scope of the searched based on entry ownership.
This is useful, if you only want to search among all publically downloadable
entries, or only among your own entries, etc.
These are the possible owner values and their meaning:
* `all`: Consider all entries.
* `public` (default): Consider all entries that can be publically downloaded,
i.e. only published entries without embargo
* `user`: Only consider entries that belong to you.
* `shared`: Only consider entries that belong to you or are shared with you.
* `visible`: Consider all entries that are visible to you. This includes
entries with embargo or unpublished entries that belong to you or are
shared with you.
* `staging`: Only search through unpublished entries.
'''
# There seems to be a slight bug in fast API. When it creates the example in OpenAPI
# it will ignore any given default or example and simply take the first enum value.
# Therefore, we put public first, which is the most default and save in most contexts.
public = 'public'
all_ = 'all'
visible = 'visible'
shared = 'shared'
user = 'user'
staging = 'staging'
admin = 'admin'
class Direction(str, enum.Enum):
'''
Order direction, either ascending (`asc`) or descending (`desc`)
'''
asc = 'asc'
desc = 'desc'
class HTTPExceptionModel(BaseModel):
detail: str
......@@ -156,36 +195,6 @@ Not.update_forward_refs()
Nested.update_forward_refs()
class Owner(str, enum.Enum):
'''
The `owner` allows to limit the scope of the searched based on entry ownership.
This is useful, if you only want to search among all publically downloadable
entries, or only among your own entries, etc.
These are the possible owner values and their meaning:
* `all`: Consider all entries.
* `public` (default): Consider all entries that can be publically downloaded,
i.e. only published entries without embargo
* `user`: Only consider entries that belong to you.
* `shared`: Only consider entries that belong to you or are shared with you.
* `visible`: Consider all entries that are visible to you. This includes
entries with embargo or unpublished entries that belong to you or are
shared with you.
* `staging`: Only search through unpublished entries.
'''
# There seems to be a slight bug in fast API. When it creates the example in OpenAPI
# it will ignore any given default or example and simply take the first enum value.
# Therefore, we put public first, which is the most default and save in most contexts.
public = 'public'
all_ = 'all'
visible = 'visible'
shared = 'shared'
user = 'user'
staging = 'staging'
admin = 'admin'
class WithQuery(BaseModel):
owner: Optional[Owner] = Body('public')
query: Optional[Query] = Body(
......@@ -424,14 +433,6 @@ class QueryParameters:
return WithQuery(query=query, owner=owner)
class Direction(str, enum.Enum):
'''
Order direction, either ascending (`asc`) or descending (`desc`)
'''
asc = 'asc'
desc = 'desc'
class MetadataRequired(BaseModel):
''' Defines which metadata quantities are included or excluded in the response. '''
......@@ -1051,6 +1052,75 @@ class Metadata(WithQueryAndPagination):
'''))
class MetadataEditListAction(BaseModel):
'''
Defines an action to perform on a list quantity. This enables users to add and remove values.
'''
op: str = Field(description=strip('''
Defines the type of operation (either `set`, `add` or `remove`)'''))
values: Union[str, List[str]] = Field(description=strip('''
The value or values to set/add/remove (string or list of strings)'''))
def _basic_type(quantity):
if quantity.type in (str, int, float, bool):
return quantity.type
return str
# Generate model for MetadataEditActions
_metadata_edit_fields = {
quantity.name: (
Optional[_basic_type(quantity)] if quantity.is_scalar
else Optional[Union[str, List[str], MetadataEditListAction]], None)
for quantity in datamodel.EditableUserMetadata.m_def.definitions
if isinstance(quantity, metainfo.Quantity)
}
MetadataEditActions = create_model('MetadataEditActions', **_metadata_edit_fields) # type: ignore
class MetadataEditRequest(WithQuery):
'''
Defines a request to edit metadata. You can specify a query (which defines
a selection of entries), an upload_id, or both. Note that if a query is specified, you can
only edit entry level metadata (like for example `comment`), not upload level metadata
(like for example changing the `upload_name` or the embargo settings), since a query
may select entries from multiple uploads, or only a subset of all the entries from an upload.
'''
upload_id: Optional[str] = Field(
description=strip('''
An optional upload_id, for restricting ourselvs to one upload. If a query is
specified, it will be automatically restricted to entries of this upload.'''))
metadata: Optional[MetadataEditActions] = Field( # type: ignore
description=strip('''
Metadata to set, on the upload and/or selected entries.'''))
entries: Optional[Dict[str, MetadataEditActions]] = Field( # type: ignore
description=strip('''
An optional dictionary, specifying metadata to set on individual entries. The field
`entries_metadata_key` defines which type of key is used in the dictionary to identify
the entries. Note, only quantities defined on the entry level can be set using this method.'''))
entries_key: Optional[str] = Field(
default='calc_id', description=strip('''
Defines which type of key is used in `entries_metadata`. Default is `calc_id`.'''))
verify_only: Optional[bool] = Field(
default=False, description=strip('''
Do not execute the request, just verifies it and provides detailed feedback on
encountered errors etc.'''))
class MetadataEditRequestResponse(MetadataEditRequest):
error: Optional[str] = Field(
description=strip('''
An error description, if the validation failed, otherwise None.'''))
feedback: Dict[str, str] = Field(
default={},
description=strip('''
A dictionary specifying strings with feedback for specific actuins.
The quantity name is used as key. The feedback may describe an error or just be
an informative message. If multiple errors occur for the same quantity, only
one error message is shown.'''))
class Files(BaseModel):
''' Configures the download of files. '''
compress: Optional[bool] = Field(
......
......@@ -95,7 +95,9 @@ def lift_embargo(dry, parallel):
upload.upload_id, upload.publish_time, embargo_length))
if not dry:
upload.set_upload_metadata({'embargo_length': 0})
upload.edit_upload_metadata(
edit_request=dict(upload_id=upload_id, metadata={'embargo_length': 0}),
user_id=config.services.admin_user_id)
return
......
......@@ -77,7 +77,7 @@ import sys
from nomad.metainfo import Environment
from .datamodel import (
Dataset, User, Author, EditableUserMetadata, UserProvidableMetadata,
Dataset, User, Author, EditableUserMetadata, UserProvidableMetadata, AuthLevel,
UploadMetadata, MongoUploadMetadata, MongoEntryMetadata, MongoSystemMetadata,
EntryMetadata, EntryArchive)
from .optimade import OptimadeEntry, Species
......
......@@ -19,6 +19,7 @@
''' All generic entry metadata and related classes. '''
from typing import List, Any
from enum import Enum
from cachetools import cached, TTLCache
from elasticsearch_dsl import analyzer, tokenizer
......@@ -40,6 +41,20 @@ from .metainfo.workflow import Workflow # noqa
from .metainfo.common_experimental import Measurement # noqa
class AuthLevel(int, Enum):
'''
Used to decorate fields with the authorization level required to edit them (using `a_auth_level`).
* `none`: No authorization required
* `coauthor`: You must be at least a coauthor of the upload to edit the field.
* `main_author`: You must be the main author of the upload to edit the field.
* `admin`: You must be admin to edit the field.
'''
none = 0
coauthor = 1
main_author = 2
admin = 3
path_analyzer = analyzer(
'path_analyzer',
tokenizer=tokenizer('path_tokenizer', 'pattern', pattern='/'))
......@@ -404,13 +419,14 @@ class EntryMetadata(metainfo.MSection):
a_elasticsearch=Elasticsearch(material_entry_type, metrics=dict(n_uploads='cardinality')))
upload_name = metainfo.Quantity(
type=str, categories=[MongoUploadMetadata],
type=str, categories=[MongoUploadMetadata, EditableUserMetadata],
description='The user provided upload name',
a_elasticsearch=Elasticsearch())
upload_create_time = metainfo.Quantity(
type=metainfo.Datetime, categories=[MongoUploadMetadata],
type=metainfo.Datetime, categories=[MongoUploadMetadata, EditableUserMetadata],
description='The date and time when the upload was created in nomad',
a_auth_level=AuthLevel.admin,
a_elasticsearch=Elasticsearch(material_entry_type))
calc_id = metainfo.Quantity(
......@@ -421,17 +437,19 @@ class EntryMetadata(metainfo.MSection):
a_elasticsearch=Elasticsearch(material_entry_type, metrics=dict(n_entries='cardinality')))
calc_hash = metainfo.Quantity(
# Note: This attribute is not stored in ES
type=str,
description='A raw file content based checksum/hash',
categories=[MongoEntryMetadata])
entry_create_time = metainfo.Quantity(
type=metainfo.Datetime, categories=[MongoEntryMetadata, MongoSystemMetadata],
type=metainfo.Datetime, categories=[MongoEntryMetadata, MongoSystemMetadata, EditableUserMetadata],
description='The date and time when the entry was created in nomad',
a_flask=dict(admin_only=True),
a_auth_level=AuthLevel.admin,
a_elasticsearch=Elasticsearch(material_entry_type))
last_edit_time = metainfo.Quantity(
# Note: This attribute is not stored in ES
type=metainfo.Datetime, categories=[MongoEntryMetadata],
description='The date and time the user metadata was last edited.')
......@@ -473,7 +491,7 @@ class EntryMetadata(metainfo.MSection):
a_elasticsearch=Elasticsearch(entry_type))
external_id = metainfo.Quantity(
type=str, categories=[MongoEntryMetadata, UserProvidableMetadata],
type=str, categories=[MongoEntryMetadata, UserProvidableMetadata, EditableUserMetadata],
description='''
A user provided external id. Usually the id for an entry in an external database
where the data was imported from.
......@@ -487,9 +505,9 @@ class EntryMetadata(metainfo.MSection):
a_elasticsearch=Elasticsearch(material_entry_type))
publish_time = metainfo.Quantity(
type=metainfo.Datetime, categories=[MongoUploadMetadata],
type=metainfo.Datetime, categories=[MongoUploadMetadata, EditableUserMetadata],
description='The date and time when the upload was published in nomad',
a_flask=dict(admin_only=True),
a_auth_level=AuthLevel.admin,
a_elasticsearch=Elasticsearch(material_entry_type))
with_embargo = metainfo.Quantity(
......@@ -498,14 +516,21 @@ class EntryMetadata(metainfo.MSection):
description='Indicated if this entry is under an embargo',
a_elasticsearch=Elasticsearch(material_entry_type))
embargo_length = metainfo.Quantity(
# Note: This attribute is not stored in ES
type=int, categories=[MongoUploadMetadata, EditableUserMetadata],
description='The length of the requested embargo period, in months')
license = metainfo.Quantity(
# Note: This attribute is not stored in ES
type=str,
description='''
A short license description (e.g. CC BY 4.0), that refers to the
license of this entry.
''',
default='CC BY 4.0',
categories=[MongoUploadMetadata, EditableUserMetadata])
categories=[MongoUploadMetadata, EditableUserMetadata],
a_auth_level=AuthLevel.admin)
processed = metainfo.Quantity(
type=bool, default=False, categories=[MongoEntryMetadata, MongoSystemMetadata],
......@@ -546,7 +571,7 @@ class EntryMetadata(metainfo.MSection):
external_db = metainfo.Quantity(
type=metainfo.MEnum('EELSDB', 'Materials Project', 'AFLOW', 'OQMD'),
categories=[MongoUploadMetadata, UserProvidableMetadata],
categories=[MongoUploadMetadata, UserProvidableMetadata, EditableUserMetadata],
description='The repository or external database where the original data resides',
a_elasticsearch=Elasticsearch(material_entry_type))
......@@ -560,11 +585,13 @@ class EntryMetadata(metainfo.MSection):
a_elasticsearch=Elasticsearch(material_entry_type))
main_author = metainfo.Quantity(
type=user_reference, categories=[MongoUploadMetadata],
type=user_reference, categories=[MongoUploadMetadata, EditableUserMetadata],
description='The main author of the entry',
a_auth_level=AuthLevel.admin,
a_elasticsearch=Elasticsearch(material_entry_type))
coauthors = metainfo.Quantity(
# Note: This attribute is not stored in ES
type=author_reference, shape=['0..*'], default=[], categories=[MongoUploadMetadata, EditableUserMetadata],
description='''
A user provided list of co-authors for the whole upload. These can view and edit the
......@@ -572,13 +599,15 @@ class EntryMetadata(metainfo.MSection):
''')
entry_coauthors = metainfo.Quantity(
type=author_reference, shape=['0..*'], default=[], categories=[MongoEntryMetadata, EditableUserMetadata],
# Note: This attribute is not stored in ES
type=author_reference, shape=['0..*'], default=[], categories=[MongoEntryMetadata],
description='''
A user provided list of co-authors specific for this entry. Note that normally,
coauthors should be set on the upload level.
A user provided list of co-authors specific for this entry. This is a legacy field,
for new uploads, coauthors should be specified on the upload level only.
''')
reviewers = metainfo.Quantity(
# Note: This attribute is not stored in ES
type=user_reference, shape=['0..*'], default=[], categories=[MongoUploadMetadata, EditableUserMetadata],
description='''
A user provided list of reviewers. Reviewers can see the whole upload, also if
......
......@@ -1356,7 +1356,10 @@ class MSection(metaclass=MObjectMeta): # TODO find a way to make this a subclas
serialize = serialize_dtype
elif isinstance(quantity_type, MEnum):
serialize = str
def serialize_enum(value):
return None if value is None else str(value)
serialize = serialize_enum
elif quantity_type == Any:
def _serialize(value: Any):
......
This diff is collapsed.
......@@ -45,14 +45,6 @@ def admin_user_auth(admin_user: User):
return create_auth_headers(admin_user)
@pytest.fixture(scope='module')
def test_users_dict(test_user, other_test_user, admin_user):
return {
'test_user': test_user,
'other_test_user': other_test_user,
'admin_user': admin_user}
@pytest.fixture(scope='module')
def test_auth_dict(
test_user, other_test_user, admin_user,
......
......@@ -17,156 +17,8 @@
#
import pytest
import math
from nomad.archive import write_partial_archive_to_mongo
from nomad.datamodel import OptimadeEntry
from nomad.processing import ProcessStatus
from tests.utils import ExampleData
@pytest.fixture(scope='session')
def client(api_v1):
return api_v1
@pytest.fixture(scope='module')
def example_data(elastic_module, raw_files_module, mongo_module, test_user, other_test_user, normalized):
'''
Provides a couple of uploads and entries including metadata, raw-data, and
archive files.
id_embargo:
1 entry, 1 material, published with embargo
id_embargo_w_coauthor:
1 entry, 1 material, published with embargo and coauthor
id_embargo_w_reviewer:
1 entry, 1 material, published with embargo and reviewer
id_unpublished:
1 entry, 1 material, unpublished
id_unpublished_w_coauthor:
1 entry, 1 material, unpublished with coauthor
id_unpublished_w_reviewer:
1 entry, 1 material, unpublished with reviewer
id_published:
23 entries, 6 materials published without embargo
partial archive exists only for id_01
raw files and archive file for id_02 are missing
id_10, id_11 reside in the same directory
id_processing:
unpublished upload without any entries, in status processing
id_empty:
unpublished upload without any entries
'''
data = ExampleData(main_author=test_user)
# 6 uploads with different combinations of main_type and sub_type
for main_type in ('embargo', 'unpublished'):
for sub_type in ('', 'w_coauthor', 'w_reviewer'):
upload_id = 'id_' + main_type + ('_' if sub_type else '') + sub_type
if main_type == 'embargo':
published = True
embargo_length = 12
upload_name = 'name_' + upload_id[3:]
else:
published = False
embargo_length = 0
upload_name = None
calc_id = upload_id + '_1'
coauthors = [other_test_user.user_id] if sub_type == 'w_coauthor' else None
reviewers = [other_test_user.user_id] if sub_type == 'w_reviewer' else None
data.create_upload(
upload_id=upload_id,
upload_name=upload_name,
coauthors=coauthors,
reviewers=reviewers,
published=published,
embargo_length=embargo_length)
data.create_entry(
upload_id=upload_id,
calc_id=calc_id,
material_id=upload_id,
mainfile=f'test_content/{calc_id}/mainfile.json')
# one upload with 23 calcs, published, no embargo
data.create_upload(
upload_id='id_published',
upload_name='name_published',
published=True)
for i in range(1, 24):
entry_id = 'id_%02d' % i
material_id = 'id_%02d' % (int(math.floor(i / 4)) + 1)
mainfile = 'test_content/subdir/test_entry_%02d/mainfile.json' % i
kwargs = dict(optimade=OptimadeEntry(nelements=2, elements=['H', 'O']))
if i == 11:
mainfile = 'test_content/subdir/test_entry_10/mainfile_11.json'
if i == 1:
kwargs['pid'] = '123'
data.create_entry(
upload_id='id_published',
calc_id=entry_id,
material_id=material_id,
mainfile=mainfile,
**kwargs)
if i == 1:
archive = data.archives[entry_id]
write_partial_archive_to_mongo(archive)
# one upload, no calcs, still processing
data.create_upload(
upload_id='id_processing',
published=False,
process_status=ProcessStatus.RUNNING)
# one upload, no calcs, unpublished
data.create_upload(
upload_id='id_empty',
published=False)
data.save(with_files=False)
del(data.archives['id_02'])
data.save(with_files=True, with_es=False, with_mongo=False)
@pytest.fixture(scope='function')
def example_data_writeable(mongo, test_user, normalized):
data = ExampleData(main_author=test_user)
# one upload with one entry, published
data.create_upload(
upload_id='id_published_w',
published=True,
embargo_length=12)
data.create_entry(
upload_id='id_published_w',
calc_id='id_published_w_entry',
mainfile='test_content/test_embargo_entry/mainfile.json')
# one upload with one entry, unpublished
data.create_upload(
upload_id='id_unpublished_w',
published=False,
embargo_length=12)
data.create_entry(
upload_id='id_unpublished_w',
calc_id='id_unpublished_w_entry',
mainfile='test_content/test_embargo_entry/mainfile.json')
# one upload, no entries, still processing
data.create_upload(
upload_id='id_processing_w',
published=False,
process_status=ProcessStatus.RUNNING)
# one upload, no entries, unpublished
data.create_upload(
upload_id='id_empty_w',
published=False)
data.save()
yield
data.delete()
......@@ -34,7 +34,7 @@ from .common import (
perform_metadata_test, post_query_test_parameters, get_query_test_parameters,
perform_owner_test, owner_test_parameters, pagination_test_parameters,
aggregation_test_parameters)
from ..conftest import example_data as data # pylint: disable=unused-import
from tests.conftest import example_data as data # pylint: disable=unused-import
'''
These are the tests for all API operations below ``entries``. The tests are organized
......
......@@ -137,10 +137,11 @@ class TestEditRepo():
def test_edit_all_properties(self, test_user, other_test_user):
edit_data = dict(
comment='test_edit_props',
references=['http://test', 'http://test2'],
# reviewers=[other_test_user.user_id], # TODO: need to set on upload level
entry_coauthors=[other_test_user.user_id])
# entry_coauthors=[other_test_user.user_id] # Not editable any more
comment='test_edit_props',
references=['http://test', 'http://test2'])
rv = self.perform_edit(**edit_data, query=self.query('upload_1'))
result = rv.json()
assert rv.status_code == 200, result
......@@ -155,18 +156,18 @@ class TestEditRepo():
assert self.mongo(1, comment='test_edit_props')
assert self.mongo(1, references=['http://test', 'http://test2'])
assert self.mongo(1, entry_coauthors=[other_test_user.user_id])
# assert self.mongo(1, entry_coauthors=[other_test_user.user_id])
# assert self.mongo(1, reviewers=[other_test_user.user_id]) TODO: need to be set on upload level
self.assert_elastic(1, comment='test_edit_props')
self.assert_elastic(1, references=['http://test', 'http://test2'])
self.assert_elastic(1, authors=[test_user.user_id, other_test_user.user_id])
self.assert_elastic(1, authors=[test_user.user_id])
# self.assert_elastic(1, viewers=[test_user.user_id, other_test_user.user_id])
edit_data = dict(
comment='',
references=[],
entry_coauthors=[])
# entry_coauthors=[]
references=[])
rv = self.perform_edit(**edit_data, query=self.query('upload_1'))
result = rv.json()
assert rv.status_code == 200
......@@ -181,7 +182,7 @@ class TestEditRepo():
assert self.mongo(1, comment=None)
assert self.mongo(1, references=[])
assert self.mongo(1, entry_coauthors=[])
# assert self.mongo(1, entry_coauthors=[])
assert self.mongo(1, reviewers=[])
self.assert_elastic(1, comment=None)
......@@ -220,19 +221,20 @@ class TestEditRepo():
assert not self.mongo(1, comment='test_edit_verify', edited=False)
def test_edit_empty_list(self, other_test_user):
rv = self.perform_edit(entry_coauthors=[other_test_user.user_id], query=self.query('upload_1'))
self.assert_edit(rv, quantity='entry_coauthors', success=True, message=False)
rv = self.perform_edit(entry_coauthors=[], query=self.query('upload_1'))
self.assert_edit(rv, quantity='entry_coauthors', success=True, message=False)