diff --git a/nomad/api.py b/nomad/api.py
index 8ae9c1c99f3ae5990ab13c0e2851ce9a40e81992..22c5e831a4a94398b9b77d35b2ac613b418cabb8 100644
--- a/nomad/api.py
+++ b/nomad/api.py
@@ -22,7 +22,7 @@ import os.path
from nomad import config
from nomad.files import UploadFile, ArchiveFile
-from nomad.utils import get_logger, create_uuid
+from nomad.utils import get_logger
from nomad.processing import Upload, NotAllowedDuringProcessing
from nomad.repo import RepoCalc
from nomad.user import User
@@ -189,8 +189,7 @@ class UploadsRes(Resource):
if json_data is None:
json_data = {}
- upload = Upload.create(
- upload_id=create_uuid(), user_id=g.user.email, name=json_data.get('name'))
+ upload = Upload.create(user=g.user, name=json_data.get('name'))
return upload.json_dict, 200
@@ -410,6 +409,7 @@ class UploadFileRes(Resource):
:status 400: if the fileformat is not supported or the form data is different than expected.
:returns: the upload (see GET /uploads/<upload_id>)
"""
+ @login_really_required
def put(self, upload_id):
logger = get_logger(__name__, endpoint='upload', action='put', upload_id=upload_id)
diff --git a/nomad/processing/data.py b/nomad/processing/data.py
index adf8f2bef33b4819c0b387e53eecfb07ba482f2b..e22a538d6288aaa882f51c800438d9bb6d6be9ad 100644
--- a/nomad/processing/data.py
+++ b/nomad/processing/data.py
@@ -36,6 +36,7 @@ from mongoengine import \
ListField, DictField, ReferenceField, IntField, connect
import mongoengine.errors
import logging
+import base64
from nomad import config, utils
from nomad.files import UploadFile, ArchiveFile, FileError
@@ -194,7 +195,6 @@ class Upload(Chord):
upload_id: the upload id generated by the database
in_staging: true if the upload is still in staging and can be edited by the uploader
is_private: true if the upload and its derivitaves are only visible to the uploader
- presigned_url: the presigned url for file upload
upload_time: the timestamp when the system realised the upload
upload_hash: the hash of the uploaded file
user_id: the id of the user that created this upload
@@ -209,12 +209,12 @@ class Upload(Chord):
in_staging = BooleanField(default=True)
is_private = BooleanField(default=False)
- presigned_url = StringField()
- upload_command = StringField()
upload_time = DateTimeField()
upload_hash = StringField(default=None)
user_id = StringField(required=True)
+ upload_url = StringField(default=None)
+ upload_command = StringField(default=None)
_initiated_parsers = IntField(default=-1)
@@ -285,11 +285,25 @@ class Upload(Chord):
Creates a new upload for the given user, a user given name is optional.
It will populate the record with a signed url and pending :class:`UploadProc`.
The upload will be already saved to the database.
+
+ Arguments:
+ user (User): The user that created the upload.
"""
+ user: User = kwargs['user']
+ del(kwargs['user'])
+ if 'upload_id' not in kwargs:
+ kwargs.update(upload_id=utils.create_uuid())
+ kwargs.update(user_id=user.email)
self = super().create(**kwargs)
- self.presigned_url = cls._external_objects_url('/uploads/%s/file' % self.upload_id)
- self.upload_command = 'curl -X put "%s" -F file=@your_local_file' % self.presigned_url
+
+ basic_auth_token = base64.b64encode(b'%s:' % user.generate_auth_token()).decode('utf-8')
+
+ self.upload_url = cls._external_objects_url('/uploads/%s/file' % self.upload_id)
+ self.upload_command = 'curl -HAuthorization: Basic %s "%s" --upload-file local_file' % (
+ basic_auth_token, self.upload_url)
+
self._continue_with('uploading')
+
return self
@property
@@ -312,7 +326,7 @@ class Upload(Chord):
'name': self.name,
'additional_metadata': self.additional_metadata,
'upload_id': self.upload_id,
- 'presigned_url': self.presigned_url,
+ 'upload_url': self.upload_url,
'upload_command': self.upload_command,
'upload_time': self.upload_time.isoformat() if self.upload_time is not None else None,
'is_stale': self.is_stale,
diff --git a/tests/processing/test_data.py b/tests/processing/test_data.py
index ecff5ab78925b16f823557bc519b6d3ad85cddca..396a93154c5a7857f339b12708bb40b187e42ee9 100644
--- a/tests/processing/test_data.py
+++ b/tests/processing/test_data.py
@@ -55,7 +55,7 @@ def uploaded_id(request, clear_files) -> Generator[str, None, None]:
def run_processing(uploaded_id: str) -> Upload:
- upload = Upload.create(upload_id=uploaded_id, user_id=me.email)
+ upload = Upload.create(upload_id=uploaded_id, user=me)
upload.upload_time = datetime.now()
assert upload.status == 'RUNNING'
diff --git a/tests/test_api.py b/tests/test_api.py
index 91dd0851ce8b6eae3933e7708d969834ca8b6ea3..580e1e59160fe26fab2154c5fb15a8d40d91e461 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -1,6 +1,7 @@
import pytest
import time
import json
+import re
from mongoengine import connect
from mongoengine.connection import disconnect
from datetime import datetime, timedelta
@@ -66,7 +67,7 @@ def assert_upload(upload_json_str, id=None, **kwargs):
if id is not None:
assert id == data['upload_id']
assert 'create_time' in data
- assert 'presigned_url' in data
+ assert 'upload_url' in data
assert 'upload_command' in data
for key, value in kwargs.items():
@@ -151,16 +152,21 @@ def test_processing(client, file, mode, worker, mocksearch, test_user_auth, no_w
upload = assert_upload(rv.data)
upload_id = upload['upload_id']
- upload_url = upload['presigned_url']
+ upload_cmd = upload['upload_command']
+ headers = dict(Authorization='Basic %s' % re.search(r'.*-HAuthorization: Basic ([^\s]+).*', upload_cmd).group(1))
upload_endpoint = '/uploads/%s' % upload_id
upload_file_endpoint = '%s/file' % upload_endpoint
+ upload_url = upload['upload_url']
assert upload_url.endswith(upload_file_endpoint)
if mode == 'multipart':
- rv = client.put(upload_file_endpoint, data=dict(file=(open(file, 'rb'), 'file')))
+ rv = client.put(
+ upload_file_endpoint,
+ data=dict(file=(open(file, 'rb'), 'file')),
+ headers=headers)
elif mode == 'stream':
with open(file, 'rb') as f:
- rv = client.put(upload_file_endpoint, data=f.read())
+ rv = client.put(upload_file_endpoint, data=f.read(), headers=headers)
else:
assert False
assert rv.status_code == 200