diff --git a/nomad/api.py b/nomad/api.py index 8ae9c1c99f3ae5990ab13c0e2851ce9a40e81992..22c5e831a4a94398b9b77d35b2ac613b418cabb8 100644 --- a/nomad/api.py +++ b/nomad/api.py @@ -22,7 +22,7 @@ import os.path from nomad import config from nomad.files import UploadFile, ArchiveFile -from nomad.utils import get_logger, create_uuid +from nomad.utils import get_logger from nomad.processing import Upload, NotAllowedDuringProcessing from nomad.repo import RepoCalc from nomad.user import User @@ -189,8 +189,7 @@ class UploadsRes(Resource): if json_data is None: json_data = {} - upload = Upload.create( - upload_id=create_uuid(), user_id=g.user.email, name=json_data.get('name')) + upload = Upload.create(user=g.user, name=json_data.get('name')) return upload.json_dict, 200 @@ -410,6 +409,7 @@ class UploadFileRes(Resource): :status 400: if the fileformat is not supported or the form data is different than expected. :returns: the upload (see GET /uploads/<upload_id>) """ + @login_really_required def put(self, upload_id): logger = get_logger(__name__, endpoint='upload', action='put', upload_id=upload_id) diff --git a/nomad/processing/data.py b/nomad/processing/data.py index adf8f2bef33b4819c0b387e53eecfb07ba482f2b..e22a538d6288aaa882f51c800438d9bb6d6be9ad 100644 --- a/nomad/processing/data.py +++ b/nomad/processing/data.py @@ -36,6 +36,7 @@ from mongoengine import \ ListField, DictField, ReferenceField, IntField, connect import mongoengine.errors import logging +import base64 from nomad import config, utils from nomad.files import UploadFile, ArchiveFile, FileError @@ -194,7 +195,6 @@ class Upload(Chord): upload_id: the upload id generated by the database in_staging: true if the upload is still in staging and can be edited by the uploader is_private: true if the upload and its derivitaves are only visible to the uploader - presigned_url: the presigned url for file upload upload_time: the timestamp when the system realised the upload upload_hash: the hash of the uploaded file user_id: the id of the user that created this upload @@ -209,12 +209,12 @@ class Upload(Chord): in_staging = BooleanField(default=True) is_private = BooleanField(default=False) - presigned_url = StringField() - upload_command = StringField() upload_time = DateTimeField() upload_hash = StringField(default=None) user_id = StringField(required=True) + upload_url = StringField(default=None) + upload_command = StringField(default=None) _initiated_parsers = IntField(default=-1) @@ -285,11 +285,25 @@ class Upload(Chord): Creates a new upload for the given user, a user given name is optional. It will populate the record with a signed url and pending :class:`UploadProc`. The upload will be already saved to the database. + + Arguments: + user (User): The user that created the upload. """ + user: User = kwargs['user'] + del(kwargs['user']) + if 'upload_id' not in kwargs: + kwargs.update(upload_id=utils.create_uuid()) + kwargs.update(user_id=user.email) self = super().create(**kwargs) - self.presigned_url = cls._external_objects_url('/uploads/%s/file' % self.upload_id) - self.upload_command = 'curl -X put "%s" -F file=@your_local_file' % self.presigned_url + + basic_auth_token = base64.b64encode(b'%s:' % user.generate_auth_token()).decode('utf-8') + + self.upload_url = cls._external_objects_url('/uploads/%s/file' % self.upload_id) + self.upload_command = 'curl -HAuthorization: Basic %s "%s" --upload-file local_file' % ( + basic_auth_token, self.upload_url) + self._continue_with('uploading') + return self @property @@ -312,7 +326,7 @@ class Upload(Chord): 'name': self.name, 'additional_metadata': self.additional_metadata, 'upload_id': self.upload_id, - 'presigned_url': self.presigned_url, + 'upload_url': self.upload_url, 'upload_command': self.upload_command, 'upload_time': self.upload_time.isoformat() if self.upload_time is not None else None, 'is_stale': self.is_stale, diff --git a/tests/processing/test_data.py b/tests/processing/test_data.py index ecff5ab78925b16f823557bc519b6d3ad85cddca..396a93154c5a7857f339b12708bb40b187e42ee9 100644 --- a/tests/processing/test_data.py +++ b/tests/processing/test_data.py @@ -55,7 +55,7 @@ def uploaded_id(request, clear_files) -> Generator[str, None, None]: def run_processing(uploaded_id: str) -> Upload: - upload = Upload.create(upload_id=uploaded_id, user_id=me.email) + upload = Upload.create(upload_id=uploaded_id, user=me) upload.upload_time = datetime.now() assert upload.status == 'RUNNING' diff --git a/tests/test_api.py b/tests/test_api.py index 91dd0851ce8b6eae3933e7708d969834ca8b6ea3..580e1e59160fe26fab2154c5fb15a8d40d91e461 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,6 +1,7 @@ import pytest import time import json +import re from mongoengine import connect from mongoengine.connection import disconnect from datetime import datetime, timedelta @@ -66,7 +67,7 @@ def assert_upload(upload_json_str, id=None, **kwargs): if id is not None: assert id == data['upload_id'] assert 'create_time' in data - assert 'presigned_url' in data + assert 'upload_url' in data assert 'upload_command' in data for key, value in kwargs.items(): @@ -151,16 +152,21 @@ def test_processing(client, file, mode, worker, mocksearch, test_user_auth, no_w upload = assert_upload(rv.data) upload_id = upload['upload_id'] - upload_url = upload['presigned_url'] + upload_cmd = upload['upload_command'] + headers = dict(Authorization='Basic %s' % re.search(r'.*-HAuthorization: Basic ([^\s]+).*', upload_cmd).group(1)) upload_endpoint = '/uploads/%s' % upload_id upload_file_endpoint = '%s/file' % upload_endpoint + upload_url = upload['upload_url'] assert upload_url.endswith(upload_file_endpoint) if mode == 'multipart': - rv = client.put(upload_file_endpoint, data=dict(file=(open(file, 'rb'), 'file'))) + rv = client.put( + upload_file_endpoint, + data=dict(file=(open(file, 'rb'), 'file')), + headers=headers) elif mode == 'stream': with open(file, 'rb') as f: - rv = client.put(upload_file_endpoint, data=f.read()) + rv = client.put(upload_file_endpoint, data=f.read(), headers=headers) else: assert False assert rv.status_code == 200