diff --git a/nomad/app/api/upload.py b/nomad/app/api/upload.py
index 6282cdc6db22a724b5924bfbbd1bfa92b7ba418b..5682aa113fa2af7113a54377ac1cc2cf21969baf 100644
--- a/nomad/app/api/upload.py
+++ b/nomad/app/api/upload.py
@@ -122,8 +122,10 @@ upload_metadata_parser = api.parser()
upload_metadata_parser.add_argument('name', type=str, help='An optional name for the upload.', location='args')
upload_metadata_parser.add_argument('local_path', type=str, help='Use a local file on the server.', location='args')
upload_metadata_parser.add_argument('token', type=str, help='Upload token to authenticate with curl command.', location='args')
+upload_metadata_parser.add_argument('oasis_upload_id', type=str, help='Use if this is an upload from an OASIS to the central NOMAD and set it to the upload_id.', location='args')
upload_metadata_parser.add_argument('file', type=FileStorage, help='The file to upload.', location='files')
+
upload_list_parser = pagination_request_parser.copy()
upload_list_parser.add_argument('state', type=str, help='List uploads with given state: all, unpublished, published.', location='args')
upload_list_parser.add_argument('name', type=str, help='Filter for uploads with the given name.', location='args')
@@ -249,8 +251,23 @@ class UploadListResource(Resource):
if Upload.user_uploads(g.user, published=False).count() >= config.services.upload_limit:
abort(400, 'Limit of unpublished uploads exceeded for user.')
+ # check if allowed to perform oasis upload
+ oasis_upload_id = request.args.get('oasis_upload_id')
+ from_oasis = oasis_upload_id is not None
+ if from_oasis is not None:
+ if not g.user.is_oasis_admin:
+ abort(401, 'Only an oasis admin can perform an oasis upload.')
+
upload_name = request.args.get('name')
- upload_id = utils.create_uuid()
+ if oasis_upload_id is not None:
+ upload_id = oasis_upload_id
+ try:
+ Upload.get(upload_id)
+ abort(400, 'An oasis upload with the given upload_id already exists.')
+ except KeyError:
+ pass
+ else:
+ upload_id = utils.create_uuid()
logger = common.logger.bind(upload_id=upload_id, upload_name=upload_name)
logger.info('upload created', )
@@ -310,7 +327,8 @@ class UploadListResource(Resource):
name=upload_name,
upload_time=datetime.utcnow(),
upload_path=upload_path,
- temporary=local_path != upload_path)
+ temporary=local_path != upload_path,
+ from_oasis=from_oasis)
upload.process_upload()
logger.info('initiated processing')
diff --git a/nomad/datamodel/datamodel.py b/nomad/datamodel/datamodel.py
index 21fa3cb541dc7d4ceaf9fdc17e16dcc5395cd961..71a316d1d448274052ad21637820073d40eaeb12 100644
--- a/nomad/datamodel/datamodel.py
+++ b/nomad/datamodel/datamodel.py
@@ -106,6 +106,8 @@ class User(Author):
is_admin = metainfo.Quantity(
type=bool, derived=lambda user: user.user_id == config.services.admin_user_id)
+ is_oasis_admin = metainfo.Quantity(type=bool, default=False)
+
@staticmethod
@cached(cache=TTLCache(maxsize=2048, ttl=24 * 3600))
def get(*args, **kwargs) -> 'User':
diff --git a/nomad/infrastructure.py b/nomad/infrastructure.py
index 7509413a9f2c0593ed6f67d8a2f5c6d7438cf32c..2115e24140ab81b358e5ac86b43bf6c0199b267e 100644
--- a/nomad/infrastructure.py
+++ b/nomad/infrastructure.py
@@ -336,12 +336,14 @@ class Keycloak():
from nomad import datamodel
kwargs = {key: value[0] for key, value in keycloak_user.get('attributes', {}).items()}
+ oasis_admin = kwargs.pop('is_oasis_admin', None) is not None
return datamodel.User(
user_id=keycloak_user['id'],
email=keycloak_user.get('email'),
username=keycloak_user.get('username'),
first_name=keycloak_user.get('firstName'),
last_name=keycloak_user.get('lastName'),
+ is_oasis_admin=oasis_admin,
created=datetime.fromtimestamp(keycloak_user['createdTimestamp'] / 1000),
**kwargs)
diff --git a/nomad/processing/data.py b/nomad/processing/data.py
index df3eb63f27777203e182d92381b88a502834032d..46c6a1eada560bd1c41be16228c78778a17f7f44 100644
--- a/nomad/processing/data.py
+++ b/nomad/processing/data.py
@@ -1208,17 +1208,13 @@ class Upload(Proc):
self.upload_files.delete()
if metadata is not None:
- self.publish_time = metadata.get('publish_time')
self.upload_time = metadata.get('upload_time')
- if self.publish_time is None:
- self.publish_time = datetime.utcnow()
- logger.warn('oasis upload without publish time')
-
if self.upload_time is None:
self.upload_time = datetime.utcnow()
logger.warn('oasis upload without upload time')
+ self.publish_time = datetime.utcnow()
self.published = True
self.last_update = datetime.utcnow()
self.save()
diff --git a/tests/app/test_api.py b/tests/app/test_api.py
index 0483280ec6f5624eac67eb88dc718cff448a4471..9c2727d8cdc9187e218de438a095989336282d96 100644
--- a/tests/app/test_api.py
+++ b/tests/app/test_api.py
@@ -171,11 +171,12 @@ class TestAuth:
assert rv.status_code == 200
data = json.loads(rv.data)
assert len(data['users'])
- keys = data['users'][0].keys()
+ user = data['users'][0]
+ keys = user.keys()
required_keys = ['name', 'email', 'user_id']
assert all(key in keys for key in required_keys)
for key in keys:
- assert data['users'][0].get(key) is not None
+ assert user.get(key) is not None
def test_invite(self, api, test_user_auth, no_warn):
rv = api.put(
@@ -530,6 +531,44 @@ class TestUploads:
rv = api.get('/raw/%s/examples_potcar/POTCAR%s.stripped' % (upload_id, ending))
assert rv.status_code == 200
+ def test_post_from_oasis_admin(self, api, other_test_user_auth, oasis_example_upload, proc_infra, no_warn):
+ rv = api.put(
+ '/uploads/?local_path=%s&oasis_upload_id=oasis_upload_id' % oasis_example_upload,
+ headers=other_test_user_auth)
+ assert rv.status_code == 401
+
+ def test_post_from_oasis_duplicate(self, api, test_user, test_user_auth, oasis_example_upload, proc_infra, no_warn):
+ Upload.create(upload_id='oasis_upload_id', user=test_user).save()
+ rv = api.put(
+ '/uploads/?local_path=%s&oasis_upload_id=oasis_upload_id' % oasis_example_upload,
+ headers=test_user_auth)
+ assert rv.status_code == 400
+
+ def test_post_from_oasis(self, api, test_user_auth, oasis_example_upload, proc_infra, no_warn):
+ rv = api.put(
+ '/uploads/?local_path=%s&oasis_upload_id=oasis_upload_id' % oasis_example_upload,
+ headers=test_user_auth)
+ assert rv.status_code == 200
+ upload = self.assert_upload(rv.data)
+ upload_id = upload['upload_id']
+ assert upload_id == 'oasis_upload_id'
+
+ # poll until completed
+ upload = self.block_until_completed(api, upload_id, test_user_auth)
+
+ assert len(upload['tasks']) == 4
+ assert upload['tasks_status'] == SUCCESS
+ assert upload['current_task'] == 'cleanup'
+ assert not upload['process_running']
+
+ upload_proc = Upload.objects(upload_id=upload_id).first()
+ assert upload_proc.published
+ assert upload_proc.from_oasis
+
+ entries = get_upload_entries_metadata(upload)
+ assert_upload_files(upload_id, entries, files.PublicUploadFiles)
+ assert_search_upload(entries, additional_keys=['atoms', 'dft.system'])
+
today = datetime.datetime.utcnow().date()
today_datetime = datetime.datetime(*today.timetuple()[:6])
diff --git a/tests/conftest.py b/tests/conftest.py
index d4ea0b531ef4ab8949177829ab2b2f1043fca433..382dbb19c8a2e9c6f83d6d1f1bb6d09d2e895b89 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -34,6 +34,7 @@ from typing import List
import json
import logging
import warnings
+import zipfile
from nomad import config, infrastructure, processing, app, utils
from nomad.datamodel import EntryArchive
@@ -227,7 +228,7 @@ def test_user_uuid(handle):
test_users = {
test_user_uuid(0): dict(username='admin', email='admin', user_id=test_user_uuid(0)),
- test_user_uuid(1): dict(username='scooper', email='sheldon.cooper@nomad-coe.eu', first_name='Sheldon', last_name='Cooper', user_id=test_user_uuid(1)),
+ test_user_uuid(1): dict(username='scooper', email='sheldon.cooper@nomad-coe.eu', first_name='Sheldon', last_name='Cooper', user_id=test_user_uuid(1), is_oasis_admin=True),
test_user_uuid(2): dict(username='lhofstadter', email='leonard.hofstadter@nomad-coe.eu', first_name='Leonard', last_name='Hofstadter', user_id=test_user_uuid(2))
}
@@ -266,7 +267,7 @@ class KeycloakMock:
def search_user(self, query):
return [
User(**test_user) for test_user in self.users.values()
- if query in ' '.join(test_user.values())]
+ if query in ' '.join([str(value) for value in test_user.values()])]
@property
def access_token(self):
@@ -576,6 +577,39 @@ def non_empty_uploaded(non_empty_example_upload: str, raw_files) -> Tuple[str, s
return example_upload_id, non_empty_example_upload
+@pytest.fixture(scope='function')
+def oasis_example_upload(non_empty_example_upload: str, raw_files) -> str:
+ processing.Upload.metadata_file_cached.cache_clear()
+
+ uploaded_path = non_empty_example_upload
+ uploaded_path_modified = os.path.join(
+ config.fs.tmp,
+ os.path.basename(non_empty_example_upload))
+ shutil.copyfile(uploaded_path, uploaded_path_modified)
+
+ metadata = {
+ 'upload_id': 'oasis_upload_id',
+ 'upload_time': '2020-01-01 00:00:00',
+ 'published': True,
+ 'entries': {
+ 'examples_template/template.json': {
+ 'calc_id': 'test_calc_id'
+ }
+ }
+ }
+
+ with zipfile.ZipFile(uploaded_path_modified, 'a') as zf:
+ with zf.open('nomad.json', 'w') as f:
+ f.write(json.dumps(metadata).encode())
+
+ return uploaded_path_modified
+
+
+@pytest.fixture(scope='function')
+def oasis_example_uploaded(oasis_example_upload: str) -> Tuple[str, str]:
+ return 'oasis_upload_id', oasis_example_upload
+
+
@pytest.mark.timeout(config.tests.default_timeout)
@pytest.fixture(scope='function')
def processed(uploaded: Tuple[str, str], test_user: User, proc_infra) -> processing.Upload:
diff --git a/tests/processing/test_data.py b/tests/processing/test_data.py
index 45aa0de5f43a9266b70c220d7be0b035eb898e6e..15d934483d04f9717f17a3f43ae3e7d1f6832b3a 100644
--- a/tests/processing/test_data.py
+++ b/tests/processing/test_data.py
@@ -22,7 +22,6 @@ from datetime import datetime
import os.path
import re
import shutil
-import json
from nomad import utils, infrastructure, config
from nomad.archive import read_partial_archive_from_mongo
@@ -205,33 +204,12 @@ def test_publish_failed(
assert_search_upload(entries, additional_keys, published=True, processed=False)
-@pytest.mark.timeout(5)
-def test_oasis_upload_processing(proc_infra, non_empty_uploaded: Tuple[str, str], test_user):
- Upload.metadata_file_cached.cache_clear()
- from shutil import copyfile
- import zipfile
-
- uploaded_id, uploaded_path = non_empty_uploaded
- uploaded_zipfile = os.path.join(config.fs.tmp, 'upload.zip')
- copyfile(uploaded_path, uploaded_zipfile)
-
- metadata = {
- 'upload_id': uploaded_id,
- 'upload_time': '2020-01-01 00:00:00',
- 'published': True,
- 'entries': {
- 'examples_template/template.json': {
- 'calc_id': 'test_calc_id'
- }
- }
- }
-
- with zipfile.ZipFile(uploaded_zipfile, 'a') as zf:
- with zf.open('nomad.json', 'w') as f:
- f.write(json.dumps(metadata).encode())
+@pytest.mark.timeout(config.tests.default_timeout)
+def test_oasis_upload_processing(proc_infra, oasis_example_uploaded: Tuple[str, str], test_user, no_warn):
+ uploaded_id, uploaded_path = oasis_example_uploaded
upload = Upload.create(
- upload_id=uploaded_id, user=test_user, upload_path=uploaded_zipfile)
+ upload_id=uploaded_id, user=test_user, upload_path=uploaded_path)
upload.from_oasis = True
assert upload.tasks_status == 'RUNNING'
@@ -241,8 +219,11 @@ def test_oasis_upload_processing(proc_infra, non_empty_uploaded: Tuple[str, str]
upload.block_until_complete(interval=.01)
assert upload.published
- assert str(upload.upload_time) == metadata['upload_time']
+ assert str(upload.upload_time) == '2020-01-01 00:00:00'
assert_processing(upload, published=True)
+ calc = Calc.objects(upload_id='oasis_upload_id').first()
+ assert calc.calc_id == 'test_calc_id'
+ assert calc.metadata['published']
@pytest.mark.timeout(config.tests.default_timeout)