From 5aded18f699b55d53d46e36ad1f2ff5e191994d5 Mon Sep 17 00:00:00 2001 From: Markus Scheidgen <markus.scheidgen@gmail.com> Date: Thu, 29 Nov 2018 15:29:26 +0100 Subject: [PATCH] Added direct upload --- nomad/api/repository.py | 2 +- nomad/api/upload.py | 68 +++++++++++++++++++++++++++++++++++++++++ tests/test_api.py | 21 +++++++++++++ 3 files changed, 90 insertions(+), 1 deletion(-) diff --git a/nomad/api/repository.py b/nomad/api/repository.py index d90a60da1a..b0a7634674 100644 --- a/nomad/api/repository.py +++ b/nomad/api/repository.py @@ -23,7 +23,7 @@ from flask_restful import Resource, abort from nomad.repo import RepoCalc -from .app import api, auth, base_path, login_if_available +from .app import api, base_path, login_if_available class RepoCalcRes(Resource): diff --git a/nomad/api/upload.py b/nomad/api/upload.py index 7cf2ce81c4..0ec2cea36d 100644 --- a/nomad/api/upload.py +++ b/nomad/api/upload.py @@ -159,6 +159,74 @@ class UploadsRes(Resource): return upload.json_dict, 200 + @login_really_required + def put(self): + """ + Upload a file and automatically create a new upload in the process. + Can be used to upload files via browser or other http clients like curl. + This will also start the processing of the upload. + + There are two basic ways to upload a file: multipart-formdata or simply streaming + the file data. Both are supported. The later one does not allow to transfer a + filename or other meta-data. If a filename is available, it will become the + name of the upload. + + .. :quickref: upload; Upload a file directly and create an upload. + + **Curl examples for both approaches**: + + .. sourcecode:: sh + + curl -X put "/nomad/api/uploads/" -F file=@local_file + curl "/nomad/api/uploads/" --upload-file local_file + + :qparam name: an optional name for the upload + :status 200: upload successfully received. + :returns: the upload (see GET /uploads/<upload_id>) + """ + # create upload + upload = Upload.create( + user=g.user, + name=request.args.get('name')) + + logger = get_logger(__name__, endpoint='upload', action='put', upload_id=upload.upload_id) + logger.info('upload created') + + uploadFile = UploadFile(upload.upload_id) + + if request.mimetype == 'application/multipart-formdata': + # multipart formdata, e.g. with curl -X put "url" -F file=@local_file + # might have performance issues for large files: https://github.com/pallets/flask/issues/2086 + if 'file' in request.files: + abort(400, message='Bad multipart-formdata, there is no file part.') + file = request.files['file'] + if upload.name is '': + upload.name = file.filename + + file.save(uploadFile.os_path) + else: + # simple streaming data in HTTP body, e.g. with curl "url" -T local_file + try: + with uploadFile.open('wb') as f: + while not request.stream.is_exhausted: + f.write(request.stream.read(1024)) + + except Exception as e: + logger.error('Error on streaming upload', exc_info=e) + abort(400, message='Some IO went wrong, download probably aborted/disrupted.') + + if not uploadFile.is_valid: + uploadFile.delete() + upload.delete() + abort(400, message='Bad file format, excpected %s.' % ", ".join(UploadFile.formats)) + + logger.info('received uploaded file') + upload.upload_time = datetime.now() + upload.process() + logger.info('initiated processing') + + return upload.json_dict, 200 + class UploadRes(Resource): """ Uploads """ diff --git a/tests/test_api.py b/tests/test_api.py index 861690acbc..4d87360984 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -288,6 +288,27 @@ def test_processing_local_path(client, file, worker, mocksearch, test_user_auth, assert_processing(client, test_user_auth, upload_id, repository_db) +@pytest.mark.parametrize('file', example_files) +@pytest.mark.parametrize('mode', ['multipart', 'stream']) +@pytest.mark.timeout(10) +def test_processing_upload(client, file, mode, worker, mocksearch, test_user_auth, no_warn, repository_db): + if mode == 'multipart': + rv = client.put( + '/uploads', + data=dict(file=(open(file, 'rb'), 'file')), + headers=test_user_auth) + elif mode == 'stream': + with open(file, 'rb') as f: + rv = client.put('/uploads', data=f.read(), headers=test_user_auth) + else: + assert False + assert rv.status_code == 200 + upload = assert_upload(rv.data) + upload_id = upload['upload_id'] + + assert_processing(client, test_user_auth, upload_id, repository_db) + + def test_repo_calc(client, example_elastic_calc, no_warn): rv = client.get( '/repo/%s/%s' % (example_elastic_calc.upload_hash, example_elastic_calc.calc_hash)) -- GitLab