Commit 5abe5bcf authored by David Sikter's avatar David Sikter Committed by David Sikter
Browse files

Fixed handling time stamps and test cases for admin attributes

parent c61e013d
......@@ -96,7 +96,7 @@ def lift_embargo(dry, parallel):
if not dry:
edit_request=dict(upload_id=upload_id, metadata={'embargo_length': 0}),
edit_request=dict(metadata={'embargo_length': 0}),
......@@ -295,9 +295,10 @@ def chown(ctx, username, uploads):
print('%d uploads selected, changing owner ...' % uploads.count())
user = datamodel.User.get(username=username)
upload_metadata = datamodel.UploadMetadata(main_author=user)
for upload in uploads:
edit_request=dict(metadata={'main_author': user.user_id}),
@uploads.command(help='Reset the processing state.')
......@@ -370,8 +370,9 @@ class MetadataEditRequestValidator:
self, definition: metainfo.Definition, value: Any) -> Any:
Verifies a *singular* action value (i.e. for list quantities we should run this method
for each value in the list, not with the list itself as input). Validates:
1) datatype,
for each value in the list, not with the list itself as input). Returns the verified
value, which may be different from the origial value. It:
1) ensures a return value of a primitive type (str, int, float, bool or None),
2) that user refs exist,
3) that datasets refs exist and do not have a doi.
4) Translates user refs to user_id and dataset refs to dataset_id, if needed.
......@@ -382,6 +383,10 @@ class MetadataEditRequestValidator:
if == 'embargo_length':
assert 0 <= value <= 36, 'Value should be between 0 and 36'
return None if value == '' else value
elif definition.type == metainfo.Datetime:
if value is not None:
datetime.fromisoformat(value) # Throws exception if badly formatted timestamp
return None if value == '' else value
elif isinstance(definition.type, metainfo.MEnum):
assert type(value) == str, 'Expected a string value'
if value == '':
......@@ -430,6 +435,8 @@ class MetadataEditRequestValidator:
definition = _editable_metadata[quantity_name]
if definition.is_scalar:
if definition.type == metainfo.Datetime and action:
return datetime.fromisoformat(action)
return action
# Non-scalar property. Verified action should be a dict with op and values
op, values = action['op'], action['values']
......@@ -512,7 +519,7 @@ class MetadataEditRequestValidator:
# We have no query. Return all entries for the upload
return Calc.objects(upload_id=upload.upload_id)
def create_response(self) -> MetadataEditRequestResponse:
def create_request_response(self) -> MetadataEditRequestResponse:
''' Creates a :class:`MetadataEditRequestResponse` with the validation results. '''
verified_dict = self.edit_request.dict()
# Overwrite input values with verified values when possible
......@@ -1959,7 +1966,7 @@ class Upload(Proc):
# Validate the request
# Create response
response = validator.create_response()
response = validator.create_request_response()
if not response.error and not edit_request.verify_only:
# Try to execute for all affected uploads
request_dict = edit_request.dict()
......@@ -1972,7 +1979,6 @@ class Upload(Proc):
# Looks good, try to trigger processing
for upload in validator.affected_uploads:
request_dict['upload_id'] = upload.upload_id
upload.edit_upload_metadata(request_dict, user.user_id) # Trigger the process
except Exception as e:
response.error = f'Failed to start process for upload {upload.upload_id}: {e}'
......@@ -1989,11 +1995,12 @@ class Upload(Proc):
logger = self.get_logger()
user = datamodel.User.get(user_id=user_id)
edit_request_obj = MetadataEditRequest(**edit_request)
# Some sanity checks, just in case
assert edit_request_obj.upload_id, 'Should specify upload_id in the edit request'
assert edit_request_obj.upload_id == self.upload_id, 'Invalid upload_id in edit_request'
assert not edit_request_obj.verify_only, 'Request has verify_only'
if 'upload_id' not in edit_request:
edit_request['upload_id'] = self.upload_id
assert edit_request['upload_id'] == self.upload_id, 'Invalid upload_id in edit_request'
assert not edit_request.get('verify_only'), 'Request has verify_only'
edit_request_obj = MetadataEditRequest(**edit_request)
# Validate the request (the @process could have been invoked directly, without previous validation)
validator = MetadataEditRequestValidator(logger, user, edit_request=edit_request_obj)
......@@ -2311,8 +2318,9 @@ class Upload(Proc):
# Validate embargo settings
if embargo_length is not None:
assert 0 <= embargo_length <= 36, 'Invalid embargo_length, must be between 0 and 36 months'
self.embargo_length = embargo_length # Set the flag also on the Upload level
self.embargo_length = embargo_length # Importing with different embargo
assert type(self.embargo_length) == int and 0 <= self.embargo_length <= 36, (
'Invalid embargo_length, must be between 0 and 36 months')
# Import the files
......@@ -16,6 +16,7 @@
# limitations under the License.
import pytest
from datetime import datetime
from nomad import datamodel, metainfo
from nomad.processing import Upload
......@@ -39,7 +40,7 @@ def assert_edit_request(user, **kwargs):
# Perform edit request
mer = MetadataEditRequest(
upload_id=upload_id, query=query, metadata=metadata, entries=entries, verify=verify)
edit_start = datetime.utcnow().isoformat()[0:22]
_response, status_code = Upload.edit_metadata(mer, user)
# Validate result
assert status_code == expected_status_code, 'Wrong status code returned'
......@@ -48,9 +49,11 @@ def assert_edit_request(user, **kwargs):
upload = Upload.get(upload_id)
for entry in upload.calcs:
assert entry.last_edit_time
assert entry.last_edit_time.isoformat()[0:22] >= edit_start
entry_metadata_mongo = entry.mongo_metadata(upload).m_to_dict()
entry_metadata_es = search(owner=None, query={'calc_id': entry.calc_id}).data[0]
values_to_check = expected_metadata.copy()
values_to_check = expected_metadata
for quantity_name, value_expected in values_to_check.items():
# Note, the expected value is provided on the "request format"
quantity = _editable_metadata[quantity_name]
......@@ -66,44 +69,54 @@ def assert_edit_request(user, **kwargs):
value_es = entry_metadata_es['writers']
elif quantity_name == 'reviewers':
value_es = entry_metadata_es['viewers']
value_es = convert_to_mongo_value(quantity, value_es, 'es', user)
value_expected = convert_to_mongo_value(quantity, value_expected, 'request', user)
# Verify value_mongo
assert value_mongo == value_expected
# Verify value_es
if quantity_name == 'coauthors':
cmp_value_mongo = convert_to_comparable_value(quantity, value_mongo, 'mongo', user)
cmp_value_es = convert_to_comparable_value(quantity, value_es, 'es', user)
cmp_value_expected = convert_to_comparable_value(quantity, value_expected, 'request', user)
# Verify mongo value
assert cmp_value_mongo == cmp_value_expected
# Verify ES value
if quantity_name == 'license':
continue # Not stored indexed by ES
elif quantity_name == 'coauthors':
# Check that writers == main_author + coauthors
assert value_es == [upload.main_author] + value_expected
assert cmp_value_es == [upload.main_author] + cmp_value_expected
elif quantity_name == 'reviewers':
# Check that viewers == main_author + coauthors + reviewers
assert set(value_es) == set(
[upload.main_author] + (upload.coauthors or []) + value_expected)
assert set(cmp_value_es) == set(
[upload.main_author] + (upload.coauthors or []) + cmp_value_expected)
assert value_es == value_expected
assert cmp_value_es == cmp_value_expected
def convert_to_mongo_value(quantity, value, from_format, user):
def convert_to_comparable_value(quantity, value, from_format, user):
Converts `value` from the given source format ('es', 'request')
to the values type used in mongo (user_id for user references, dataset_id
for datasets, etc). List quantities are also guaranteed to be converted to lists.
Converts `value` from the given source format ('mongo', 'es', 'request')
to a value that can be compared (user_id for user references, dataset_id
for datasets, timestamp strings with no more than millisecond precision, etc).
List quantities are also guaranteed to be converted to lists.
if quantity.is_scalar:
return convert_to_mongo_value_single(quantity, value, from_format, user)
return convert_to_comparable_value_single(quantity, value, from_format, user)
if type(value) != list:
value = [value]
return [convert_to_mongo_value_single(quantity, v, from_format, user) for v in value]
return [convert_to_comparable_value_single(quantity, v, from_format, user) for v in value]
def convert_to_mongo_value_single(quantity, value, format, user):
def convert_to_comparable_value_single(quantity, value, format, user):
if quantity.type in (str, int, float, bool) or isinstance(quantity.type, metainfo.MEnum):
if value == '' and format == 'request':
return None
return value
elif quantity.type == metainfo.Datetime:
if not value:
return None
return value[0:22] # Only compare to the millisecond level (mongo's maximal precision)
elif isinstance(quantity.type, metainfo.Reference):
# Should be reference
verify_reference = quantity.type.target_section_def.section_cls
if verify_reference in [datamodel.User, datamodel.Author]:
if format == 'mongo':
return value
if format == 'es':
return value['user_id']
elif format == 'request':
......@@ -115,7 +128,9 @@ def convert_to_mongo_value_single(quantity, value, format, user):
except KeyError:
return datamodel.User.get(email=value).user_id
elif verify_reference == datamodel.Dataset:
if format == 'es':
if format == 'mongo':
return value
elif format == 'es':
return value['dataset_id']
elif format == 'request':
......@@ -184,3 +199,20 @@ def test_set_and_clear_all(proc_infra, example_data_writeable, a_dataset, test_u
def test_admin_quantities(proc_infra, example_data_writeable, test_user, other_test_user, admin_user):
now = datetime.utcnow()
now_iso = now.isoformat()
admin_fields = dict(
license='a license',
user=admin_user, upload_id='id_published_w', metadata=admin_fields)
# try to do the same as a non-admin
for k, v in admin_fields.items():
user=test_user, upload_id='id_published_w', metadata={k: v}, expected_status_code=401)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment