Commit 82220c0b authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added tests for unstage/commit with (migration) metadata.

parent 9b1103aa
......@@ -47,7 +47,7 @@ class AdminOperationsResource(Resource):
Reset and remove can be disabled.
"""
if g.user.email != 'admin':
if not g.user.is_admin:
abort(401, message='Only the admin user can perform this operation.')
if operation == 'reset':
......
......@@ -42,7 +42,8 @@ CORS(app)
api = Api(
app, version='1.0', title='nomad@FAIRDI API',
description='Official API for nomad@FAIRDI services.')
description='Official API for nomad@FAIRDI services.',
validate=True)
""" Provides the flask restplust api instance """
......
......@@ -93,11 +93,15 @@ meta_data_model = api.model('MetaData', {
'comment': fields.List(fields.String, description='The comment are shown in the repository for each calculation.'),
'references': fields.List(fields.String, descriptions='References allow to link calculations to external source, e.g. URLs.'),
'coauthors': fields.List(fields.String, description='A list of co-authors given by user_id.'),
'share_with': fields.List(fields.String, description='A list of users to share calculations with given by user_id.')
'shared_with': fields.List(fields.String, description='A list of users to share calculations with given by user_id.'),
'_upload_time': fields.List(fields.DateTime(dt_format='iso8601'), description='Overrride the upload time.'),
'_uploader': fields.List(fields.String, description='Override the uploader with the given user id.')
})
calc_meta_data_model = api.inherit('CalcMetaData', meta_data_model, {
'mainfile': fields.String(description='The calculation main output file is used to identify the calculation in the upload.')
'mainfile': fields.String(description='The calculation main output file is used to identify the calculation in the upload.'),
'_checksum': fields.String(description='Override the calculation checksum'),
'_pid': fields.String(description='Assign a specific pid. It must be unique.')
})
upload_meta_data_model = api.inherit('UploadMetaData', meta_data_model, {
......@@ -226,7 +230,7 @@ class UploadResource(Resource):
except KeyError:
abort(404, message='Upload with id %s does not exist.' % upload_id)
if upload.user_id != str(g.user.user_id):
if upload.user_id != str(g.user.user_id) and not g.user.is_admin:
abort(404, message='Upload with id %s does not exist.' % upload_id)
try:
......@@ -276,7 +280,7 @@ class UploadResource(Resource):
except KeyError:
abort(404, message='Upload with id %s does not exist.' % upload_id)
if upload.user_id != str(g.user.user_id):
if upload.user_id != str(g.user.user_id) and not g.user.is_admin:
abort(404, message='Upload with id %s does not exist.' % upload_id)
if not upload.in_staging:
......@@ -291,6 +295,7 @@ class UploadResource(Resource):
@api.doc('exec_upload_command')
@api.response(404, 'Upload does not exist or is not allowed')
@api.response(400, 'Operation is not supported')
@api.response(401, 'If the operation is not allowed for the current user')
@api.marshal_with(upload_model, skip_none=True, code=200, description='Upload unstaged successfully')
@api.expect(upload_operation_model)
@login_really_required
......@@ -299,7 +304,8 @@ class UploadResource(Resource):
Execute an upload operation. Available operations: ``unstage``
Unstage accepts further meta data that allows to provide coauthors, comments,
external references, etc.
external references, etc. See the model for details. The fields that start with
``_underscore`` are only available for users with administrative priviledges.
Unstage changes the visibility of the upload. Clients can specify the visibility
via meta data.
......@@ -309,7 +315,7 @@ class UploadResource(Resource):
except KeyError:
abort(404, message='Upload with id %s does not exist.' % upload_id)
if upload.user_id != str(g.user.user_id):
if upload.user_id != str(g.user.user_id) and not g.user.is_admin:
abort(404, message='Upload with id %s does not exist.' % upload_id)
json_data = request.get_json()
......@@ -317,7 +323,14 @@ class UploadResource(Resource):
json_data = {}
operation = json_data.get('operation')
meta_data = json_data.get('meta_data', {})
for key in meta_data:
if key.startswith('_'):
if not g.user.is_admin:
abort(401, message='Only admin users can use _meta_data_keys.')
break
if operation == 'unstage':
if not upload.in_staging:
abort(400, message='Operation not allowed, upload is not in staging.')
......
......@@ -36,7 +36,6 @@ This module also provides functionality to add parsed calculation data to the db
"""
import itertools
import enum
from passlib.hash import bcrypt
from sqlalchemy import Column, Integer, String, Boolean, DateTime, ForeignKey, Enum
from sqlalchemy.orm import relationship
......@@ -158,7 +157,7 @@ def add_calculation(upload, coe_upload, calc: RepoCalc, calc_meta_data: dict) ->
user_metadata = UserMetaData(
calc=coe_calc,
label=calc_meta_data.get('comment', None),
permission=1 if calc_meta_data.get('restricted', False) else 0)
permission=(1 if calc_meta_data.get('with_embargo', False) else 0))
repo_db.add(user_metadata)
spacegroup = Spacegroup(
......@@ -177,7 +176,7 @@ def add_calculation(upload, coe_upload, calc: RepoCalc, calc_meta_data: dict) ->
coe_calc.set_value(topic_basis_set_type, calc.basis_set_type)
# user relations
owner_user_id = calc_meta_data.get('uploader', int(upload.user_id))
owner_user_id = calc_meta_data.get('_uploader', int(upload.user_id))
ownership = Ownership(calc_id=coe_calc.calc_id, user_id=owner_user_id)
repo_db.add(ownership)
......@@ -185,29 +184,28 @@ def add_calculation(upload, coe_upload, calc: RepoCalc, calc_meta_data: dict) ->
coauthorship = CoAuthorship(calc_id=coe_calc.calc_id, user_id=int(coauthor_id))
repo_db.add(coauthorship)
for shared_with_id in calc_meta_data.get('share_with', []):
for shared_with_id in calc_meta_data.get('shared_with', []):
shareship = Shareship(calc_id=coe_calc.calc_id, user_id=int(shared_with_id))
repo_db.add(shareship)
# datasets
for dataset_id in calc_meta_data.get('_datasets', []):
for dataset_id in calc_meta_data.get('datasets', []):
dataset = CalcSet(parent_calc_id=dataset_id, children_calc_id=coe_calc.calc_id)
repo_db.add(dataset)
# references
for reference in calc_meta_data.get('references', []):
citation = repo_db.query(Citation).filter_by(
label=reference,
kind=CitationKind.EXTERNAL).first()
value=reference,
kind='EXTERNAL').first()
if citation is None:
citation = Citation(label=reference, kind=CitationKind.EXTERNAL)
citation = Citation(value=reference, kind='EXTERNAL')
repo_db.add(citation)
metadata_citation = MetaDataCitation(
calc_id=coe_calc.calc_id,
citation_id=citation.citation_id)
citation=citation)
repo_db.add(metadata_citation)
......@@ -352,24 +350,20 @@ class CalcSet(Base): # type: ignore
children_calc_id = Column(Integer, ForeignKey('calculations.calc_id'), primary_key=True)
class MetaDataCitation(Base): # type: ignore
__tablename__ = 'metadata_citations'
calc_id = Column(Integer, ForeignKey('calculations.calc_id'), primary_key=True)
citation_id = Column(Integer, ForeignKey('citations.citation_id'), primary_key=True)
class CitationKind(enum.Enum):
INTERNAL = 1
EXTERNAL = 2
class Citation(Base): # type: ignore
__tablename__ = 'citations'
citation_id = Column(Integer, primary_key=True)
value = Column(String)
kind = Column(Enum(CitationKind))
kind = Column(Enum('INTERNAL', 'EXTERNAL', name='citation_kind_enum'))
class MetaDataCitation(Base): # type: ignore
__tablename__ = 'metadata_citations'
calc_id = Column(Integer, ForeignKey('calculations.calc_id'), primary_key=True)
citation_id = Column(Integer, ForeignKey('citations.citation_id'), primary_key=True)
citation = relationship('Citation')
class LoginException(Exception):
......@@ -414,6 +408,10 @@ class User(Base): # type: ignore
return session.token.encode('utf-8')
@property
def is_admin(self) -> bool:
return self.email == 'admin'
@staticmethod
def verify_user_password(email, password):
repo_db = infrastructure.repository_db
......
......@@ -117,6 +117,12 @@ def repository_db(monkeysession):
session.close()
@pytest.fixture(scope='function')
def clean_repository_db(repository_db):
repository_db.execute('TRUNCATE uploads CASCADE;')
yield repository_db
@pytest.fixture(scope='session')
def test_user(repository_db):
from nomad import coe_repo
......
......@@ -77,12 +77,13 @@ def test_other_user_auth(other_test_user: User):
return create_auth_headers(other_test_user)
class TestAdmin:
@pytest.fixture(scope='session')
def admin_user_auth(self, admin_user: User):
@pytest.fixture(scope='session')
def admin_user_auth(admin_user: User):
return create_auth_headers(admin_user)
class TestAdmin:
@pytest.mark.timeout(10)
def test_reset(self, client, admin_user_auth, repository_db):
rv = client.post('/admin/reset', headers=admin_user_auth)
......@@ -209,11 +210,11 @@ class TestUploads:
upload = self.assert_upload(rv.data)
assert len(upload['calcs']['results']) == 1
def assert_unstage(self, client, test_user_auth, upload_id, proc_infra):
def assert_unstage(self, client, test_user_auth, upload_id, proc_infra, meta_data={}):
rv = client.post(
'/uploads/%s' % upload_id,
headers=test_user_auth,
data=json.dumps(dict(operation='unstage')),
data=json.dumps(dict(operation='unstage', meta_data=meta_data)),
content_type='application/json')
assert rv.status_code == 200
rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
......@@ -221,10 +222,9 @@ class TestUploads:
upload = self.assert_upload(rv.data)
empty_upload = upload['calcs']['pagination']['total'] == 0
rv = client.get('/uploads/', headers=test_user_auth)
assert rv.status_code == 200
self.assert_uploads(rv.data, count=0)
assert_coe_upload(upload['upload_hash'], proc_infra['repository_db'], empty=empty_upload)
assert_coe_upload(
upload['upload_hash'], proc_infra['repository_db'],
empty=empty_upload, meta_data=meta_data)
def test_get_command(self, client, test_user_auth, no_warn):
rv = client.get('/uploads/command', headers=test_user_auth)
......@@ -285,7 +285,7 @@ class TestUploads:
assert rv.status_code == 400
self.assert_processing(client, test_user_auth, upload['upload_id'])
def test_delete_unstaged(self, client, test_user_auth, proc_infra):
def test_delete_unstaged(self, client, test_user_auth, proc_infra, clean_repository_db):
rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
upload = self.assert_upload(rv.data)
self.assert_processing(client, test_user_auth, upload['upload_id'])
......@@ -301,12 +301,45 @@ class TestUploads:
assert rv.status_code == 200
@pytest.mark.parametrize('example_file', example_files)
def test_post(self, client, test_user_auth, example_file, proc_infra):
def test_post(self, client, test_user_auth, example_file, proc_infra, clean_repository_db):
rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
upload = self.assert_upload(rv.data)
self.assert_processing(client, test_user_auth, upload['upload_id'])
self.assert_unstage(client, test_user_auth, upload['upload_id'], proc_infra)
def test_post_metadata(
self, client, proc_infra, admin_user_auth, test_user_auth, test_user,
other_test_user, clean_repository_db):
rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
upload = self.assert_upload(rv.data)
self.assert_processing(client, test_user_auth, upload['upload_id'])
meta_data = dict(comment='test comment')
self.assert_unstage(client, admin_user_auth, upload['upload_id'], proc_infra, meta_data)
def test_post_metadata_forbidden(self, client, proc_infra, test_user_auth, clean_repository_db):
rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
upload = self.assert_upload(rv.data)
self.assert_processing(client, test_user_auth, upload['upload_id'])
rv = client.post(
'/uploads/%s' % upload['upload_id'],
headers=test_user_auth,
data=json.dumps(dict(operation='unstage', meta_data=dict(_pid=256))),
content_type='application/json')
assert rv.status_code == 401
# TODO validate metadata (or all input models in API for that matter)
# def test_post_bad_metadata(self, client, proc_infra, test_user_auth, clean_repository_db):
# rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
# upload = self.assert_upload(rv.data)
# self.assert_processing(client, test_user_auth, upload['upload_id'])
# rv = client.post(
# '/uploads/%s' % upload['upload_id'],
# headers=test_user_auth,
# data=json.dumps(dict(operation='unstage', meta_data=dict(doesnotexist='hi'))),
# content_type='application/json')
# assert rv.status_code == 400
class TestRepo:
def test_calc(self, client, example_elastic_calc, no_warn):
......
......@@ -14,8 +14,10 @@
import pytest
import json
import datetime
from nomad.coe_repo import User, Calc, CalcMetaData, StructRatio, Upload, add_upload
from nomad.coe_repo import User, Calc, CalcMetaData, StructRatio, Upload, add_upload, \
UserMetaData, Citation, MetaDataCitation, Shareship, CoAuthorship, Ownership
from tests.processing.test_data import processed_upload # pylint: disable=unused-import
from tests.processing.test_data import uploaded_id # pylint: disable=unused-import
......@@ -39,33 +41,126 @@ def test_password_authorize(test_user):
assert_user(user, test_user)
def assert_coe_upload(upload_hash, repository_db, empty=False):
coe_upload = repository_db.query(Upload).filter_by(upload_name=upload_hash).first()
def assert_coe_upload(upload_hash, repository_db, empty=False, meta_data={}):
coe_uploads = repository_db.query(Upload).filter_by(upload_name=upload_hash)
if empty:
assert coe_upload is None
assert coe_uploads.count() == 0
else:
assert coe_upload is not None
assert coe_uploads.count() == 1
coe_upload = coe_uploads.first()
coe_upload_id = coe_upload.upload_id
one_calc_exist = False
for calc in repository_db.query(Calc).filter_by(origin_id=coe_upload_id):
one_calc_exist = True
assert calc.origin_id == coe_upload_id
metadata = repository_db.query(CalcMetaData).filter_by(calc_id=calc.calc_id).first()
assert metadata is not None
assert metadata.chemical_formula is not None
filenames = metadata.filenames.decode('utf-8')
assert_coe_calc(calc, repository_db, meta_data=meta_data)
if '_upload_time' in meta_data:
assert coe_upload.created.isoformat()[:26] == meta_data['_upload_time']
assert one_calc_exist
def assert_coe_calc(calc, repository_db, meta_data={}):
calc_id = calc.calc_id
calc_meta_data = repository_db.query(CalcMetaData).filter_by(calc_id=calc_id).first()
assert calc_meta_data is not None
assert calc_meta_data.calc is not None
assert calc_meta_data.chemical_formula is not None
filenames = calc_meta_data.filenames.decode('utf-8')
assert len(json.loads(filenames)) == 5
struct_ratio = repository_db.query(StructRatio).filter_by(calc_id=calc.calc_id).first()
# struct ratio
struct_ratio = repository_db.query(StructRatio).filter_by(calc_id=calc_id).first()
assert struct_ratio is not None
assert struct_ratio.chemical_formula == metadata.chemical_formula
assert struct_ratio.chemical_formula == calc_meta_data.chemical_formula
assert struct_ratio.formula_units == 1
# pid
if '_pid' in meta_data:
assert calc_id == int(meta_data['_pid'])
# checksum
if '_checksum' in meta_data:
calc.checksum == meta_data['_checksum']
# comments
comment = repository_db.query(UserMetaData).filter_by(
label=meta_data.get('comment', 'not existing comment'),
calc_id=calc_id).first()
if 'comment' in meta_data:
assert comment is not None
else:
assert comment is None
# references
if 'references' in meta_data:
for reference in meta_data['references']:
citation = repository_db.query(Citation).filter_by(
value=reference, kind='EXTERNAL').first()
assert citation is not None
assert repository_db.query(MetaDataCitation).filter_by(
citation_id=citation.citation_id, calc_id=calc_id).first() is not None
else:
repository_db.query(MetaDataCitation).filter_by(calc_id=calc_id).first() is None
# coauthors
if 'coauthors' in meta_data:
for coauthor in meta_data['coauthors']:
assert repository_db.query(CoAuthorship).filter_by(
user_id=coauthor, calc_id=calc_id).first() is not None
else:
assert repository_db.query(CoAuthorship).filter_by(calc_id=calc_id).first() is None
# coauthors
if 'shared_with' in meta_data:
for coauthor in meta_data['shared_with']:
assert repository_db.query(Shareship).filter_by(
user_id=coauthor, calc_id=calc_id).first() is not None
else:
assert repository_db.query(Shareship).filter_by(calc_id=calc_id).first() is None
# ownership
owners = repository_db.query(Ownership).filter_by(calc_id=calc_id)
assert owners.count() == 1
if '_uploader' in meta_data:
assert owners.first().user_id == meta_data['_uploader']
# embargo/restriction/permission
user_meta_data = repository_db.query(UserMetaData).filter_by(
calc_id=calc_meta_data.calc_id).first()
assert user_meta_data is not None
assert user_meta_data.permission == (1 if meta_data.get('with_embargo', False) else 0)
@pytest.mark.timeout(10)
def test_add_upload(clean_repository_db, processed_upload):
empty = processed_upload.total_calcs == 0
processed_upload.upload_hash = str(1)
add_upload(processed_upload)
assert_coe_upload(processed_upload.upload_hash, clean_repository_db, empty=empty)
processed_upload.upload_hash = str(2)
add_upload(processed_upload)
assert_coe_upload(processed_upload.upload_hash, clean_repository_db, empty=empty)
@pytest.mark.timeout(10)
def test_add_upload(repository_db, processed_upload):
coe_upload_id = add_upload(processed_upload)
if coe_upload_id:
assert_coe_upload(processed_upload.upload_hash, repository_db)
coe_upload_id = add_upload(processed_upload)
if coe_upload_id:
assert_coe_upload(processed_upload.upload_hash, repository_db)
def test_add_upload_metadata(clean_repository_db, processed_upload, other_test_user, test_user):
empty = processed_upload.total_calcs == 0
meta_data = {
'comment': 'test comment',
'with_embargo': True,
'references': ['http://external.ref/one', 'http://external.ref/two'],
'_uploader': other_test_user.user_id,
'coauthors': [test_user.user_id],
'_checksum': 1,
'_upload_time': datetime.datetime.now().isoformat(),
'_pid': 256
}
add_upload(processed_upload, meta_data=meta_data)
assert_coe_upload(processed_upload.upload_hash, clean_repository_db, empty=empty, meta_data=meta_data)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment