Commit 5dca758c authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added osis uploads to API.

parent 12b7c869
Pipeline #89212 passed with stages
in 24 minutes and 8 seconds
......@@ -122,8 +122,10 @@ upload_metadata_parser = api.parser()
upload_metadata_parser.add_argument('name', type=str, help='An optional name for the upload.', location='args')
upload_metadata_parser.add_argument('local_path', type=str, help='Use a local file on the server.', location='args')
upload_metadata_parser.add_argument('token', type=str, help='Upload token to authenticate with curl command.', location='args')
upload_metadata_parser.add_argument('oasis_upload_id', type=str, help='Use if this is an upload from an OASIS to the central NOMAD and set it to the upload_id.', location='args')
upload_metadata_parser.add_argument('file', type=FileStorage, help='The file to upload.', location='files')
upload_list_parser = pagination_request_parser.copy()
upload_list_parser.add_argument('state', type=str, help='List uploads with given state: all, unpublished, published.', location='args')
upload_list_parser.add_argument('name', type=str, help='Filter for uploads with the given name.', location='args')
......@@ -249,8 +251,23 @@ class UploadListResource(Resource):
if Upload.user_uploads(g.user, published=False).count() >= config.services.upload_limit:
abort(400, 'Limit of unpublished uploads exceeded for user.')
# check if allowed to perform oasis upload
oasis_upload_id = request.args.get('oasis_upload_id')
from_oasis = oasis_upload_id is not None
if from_oasis is not None:
if not g.user.is_oasis_admin:
abort(401, 'Only an oasis admin can perform an oasis upload.')
upload_name = request.args.get('name')
upload_id = utils.create_uuid()
if oasis_upload_id is not None:
upload_id = oasis_upload_id
try:
Upload.get(upload_id)
abort(400, 'An oasis upload with the given upload_id already exists.')
except KeyError:
pass
else:
upload_id = utils.create_uuid()
logger = common.logger.bind(upload_id=upload_id, upload_name=upload_name)
logger.info('upload created', )
......@@ -310,7 +327,8 @@ class UploadListResource(Resource):
name=upload_name,
upload_time=datetime.utcnow(),
upload_path=upload_path,
temporary=local_path != upload_path)
temporary=local_path != upload_path,
from_oasis=from_oasis)
upload.process_upload()
logger.info('initiated processing')
......
......@@ -106,6 +106,8 @@ class User(Author):
is_admin = metainfo.Quantity(
type=bool, derived=lambda user: user.user_id == config.services.admin_user_id)
is_oasis_admin = metainfo.Quantity(type=bool, default=False)
@staticmethod
@cached(cache=TTLCache(maxsize=2048, ttl=24 * 3600))
def get(*args, **kwargs) -> 'User':
......
......@@ -336,12 +336,14 @@ class Keycloak():
from nomad import datamodel
kwargs = {key: value[0] for key, value in keycloak_user.get('attributes', {}).items()}
oasis_admin = kwargs.pop('is_oasis_admin', None) is not None
return datamodel.User(
user_id=keycloak_user['id'],
email=keycloak_user.get('email'),
username=keycloak_user.get('username'),
first_name=keycloak_user.get('firstName'),
last_name=keycloak_user.get('lastName'),
is_oasis_admin=oasis_admin,
created=datetime.fromtimestamp(keycloak_user['createdTimestamp'] / 1000),
**kwargs)
......
......@@ -1208,17 +1208,13 @@ class Upload(Proc):
self.upload_files.delete()
if metadata is not None:
self.publish_time = metadata.get('publish_time')
self.upload_time = metadata.get('upload_time')
if self.publish_time is None:
self.publish_time = datetime.utcnow()
logger.warn('oasis upload without publish time')
if self.upload_time is None:
self.upload_time = datetime.utcnow()
logger.warn('oasis upload without upload time')
self.publish_time = datetime.utcnow()
self.published = True
self.last_update = datetime.utcnow()
self.save()
......
......@@ -171,11 +171,12 @@ class TestAuth:
assert rv.status_code == 200
data = json.loads(rv.data)
assert len(data['users'])
keys = data['users'][0].keys()
user = data['users'][0]
keys = user.keys()
required_keys = ['name', 'email', 'user_id']
assert all(key in keys for key in required_keys)
for key in keys:
assert data['users'][0].get(key) is not None
assert user.get(key) is not None
def test_invite(self, api, test_user_auth, no_warn):
rv = api.put(
......@@ -530,6 +531,44 @@ class TestUploads:
rv = api.get('/raw/%s/examples_potcar/POTCAR%s.stripped' % (upload_id, ending))
assert rv.status_code == 200
def test_post_from_oasis_admin(self, api, other_test_user_auth, oasis_example_upload, proc_infra, no_warn):
rv = api.put(
'/uploads/?local_path=%s&oasis_upload_id=oasis_upload_id' % oasis_example_upload,
headers=other_test_user_auth)
assert rv.status_code == 401
def test_post_from_oasis_duplicate(self, api, test_user, test_user_auth, oasis_example_upload, proc_infra, no_warn):
Upload.create(upload_id='oasis_upload_id', user=test_user).save()
rv = api.put(
'/uploads/?local_path=%s&oasis_upload_id=oasis_upload_id' % oasis_example_upload,
headers=test_user_auth)
assert rv.status_code == 400
def test_post_from_oasis(self, api, test_user_auth, oasis_example_upload, proc_infra, no_warn):
rv = api.put(
'/uploads/?local_path=%s&oasis_upload_id=oasis_upload_id' % oasis_example_upload,
headers=test_user_auth)
assert rv.status_code == 200
upload = self.assert_upload(rv.data)
upload_id = upload['upload_id']
assert upload_id == 'oasis_upload_id'
# poll until completed
upload = self.block_until_completed(api, upload_id, test_user_auth)
assert len(upload['tasks']) == 4
assert upload['tasks_status'] == SUCCESS
assert upload['current_task'] == 'cleanup'
assert not upload['process_running']
upload_proc = Upload.objects(upload_id=upload_id).first()
assert upload_proc.published
assert upload_proc.from_oasis
entries = get_upload_entries_metadata(upload)
assert_upload_files(upload_id, entries, files.PublicUploadFiles)
assert_search_upload(entries, additional_keys=['atoms', 'dft.system'])
today = datetime.datetime.utcnow().date()
today_datetime = datetime.datetime(*today.timetuple()[:6])
......
......@@ -34,6 +34,7 @@ from typing import List
import json
import logging
import warnings
import zipfile
from nomad import config, infrastructure, processing, app, utils
from nomad.datamodel import EntryArchive
......@@ -227,7 +228,7 @@ def test_user_uuid(handle):
test_users = {
test_user_uuid(0): dict(username='admin', email='admin', user_id=test_user_uuid(0)),
test_user_uuid(1): dict(username='scooper', email='sheldon.cooper@nomad-coe.eu', first_name='Sheldon', last_name='Cooper', user_id=test_user_uuid(1)),
test_user_uuid(1): dict(username='scooper', email='sheldon.cooper@nomad-coe.eu', first_name='Sheldon', last_name='Cooper', user_id=test_user_uuid(1), is_oasis_admin=True),
test_user_uuid(2): dict(username='lhofstadter', email='leonard.hofstadter@nomad-coe.eu', first_name='Leonard', last_name='Hofstadter', user_id=test_user_uuid(2))
}
......@@ -266,7 +267,7 @@ class KeycloakMock:
def search_user(self, query):
return [
User(**test_user) for test_user in self.users.values()
if query in ' '.join(test_user.values())]
if query in ' '.join([str(value) for value in test_user.values()])]
@property
def access_token(self):
......@@ -576,6 +577,39 @@ def non_empty_uploaded(non_empty_example_upload: str, raw_files) -> Tuple[str, s
return example_upload_id, non_empty_example_upload
@pytest.fixture(scope='function')
def oasis_example_upload(non_empty_example_upload: str, raw_files) -> str:
processing.Upload.metadata_file_cached.cache_clear()
uploaded_path = non_empty_example_upload
uploaded_path_modified = os.path.join(
config.fs.tmp,
os.path.basename(non_empty_example_upload))
shutil.copyfile(uploaded_path, uploaded_path_modified)
metadata = {
'upload_id': 'oasis_upload_id',
'upload_time': '2020-01-01 00:00:00',
'published': True,
'entries': {
'examples_template/template.json': {
'calc_id': 'test_calc_id'
}
}
}
with zipfile.ZipFile(uploaded_path_modified, 'a') as zf:
with zf.open('nomad.json', 'w') as f:
f.write(json.dumps(metadata).encode())
return uploaded_path_modified
@pytest.fixture(scope='function')
def oasis_example_uploaded(oasis_example_upload: str) -> Tuple[str, str]:
return 'oasis_upload_id', oasis_example_upload
@pytest.mark.timeout(config.tests.default_timeout)
@pytest.fixture(scope='function')
def processed(uploaded: Tuple[str, str], test_user: User, proc_infra) -> processing.Upload:
......
......@@ -22,7 +22,6 @@ from datetime import datetime
import os.path
import re
import shutil
import json
from nomad import utils, infrastructure, config
from nomad.archive import read_partial_archive_from_mongo
......@@ -205,33 +204,12 @@ def test_publish_failed(
assert_search_upload(entries, additional_keys, published=True, processed=False)
@pytest.mark.timeout(5)
def test_oasis_upload_processing(proc_infra, non_empty_uploaded: Tuple[str, str], test_user):
Upload.metadata_file_cached.cache_clear()
from shutil import copyfile
import zipfile
uploaded_id, uploaded_path = non_empty_uploaded
uploaded_zipfile = os.path.join(config.fs.tmp, 'upload.zip')
copyfile(uploaded_path, uploaded_zipfile)
metadata = {
'upload_id': uploaded_id,
'upload_time': '2020-01-01 00:00:00',
'published': True,
'entries': {
'examples_template/template.json': {
'calc_id': 'test_calc_id'
}
}
}
with zipfile.ZipFile(uploaded_zipfile, 'a') as zf:
with zf.open('nomad.json', 'w') as f:
f.write(json.dumps(metadata).encode())
@pytest.mark.timeout(config.tests.default_timeout)
def test_oasis_upload_processing(proc_infra, oasis_example_uploaded: Tuple[str, str], test_user, no_warn):
uploaded_id, uploaded_path = oasis_example_uploaded
upload = Upload.create(
upload_id=uploaded_id, user=test_user, upload_path=uploaded_zipfile)
upload_id=uploaded_id, user=test_user, upload_path=uploaded_path)
upload.from_oasis = True
assert upload.tasks_status == 'RUNNING'
......@@ -241,8 +219,11 @@ def test_oasis_upload_processing(proc_infra, non_empty_uploaded: Tuple[str, str]
upload.block_until_complete(interval=.01)
assert upload.published
assert str(upload.upload_time) == metadata['upload_time']
assert str(upload.upload_time) == '2020-01-01 00:00:00'
assert_processing(upload, published=True)
calc = Calc.objects(upload_id='oasis_upload_id').first()
assert calc.calc_id == 'test_calc_id'
assert calc.metadata['published']
@pytest.mark.timeout(config.tests.default_timeout)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment