Commit c8f69eb5 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added upload.delete in API.

parent f54e1102
from typing import Tuple
from flask import Flask, request, redirect
from flask_restful import Resource, Api, abort
from datetime import datetime
......@@ -39,11 +40,7 @@ def _external_objects_url(url):
'%s%s%s' % (config.services.objects_host, port_with_colon, config.services.objects_base_path))
def _update_and_render(upload: users.Upload):
"""
If the given upload as a processing state attached, it will attempt to update this
state and store the results, before the upload is rendered for the client.
"""
def _updated_proc(upload: users.Upload) -> Tuple[UploadProc, bool]:
is_stale = False
if upload.proc:
......@@ -58,6 +55,10 @@ def _update_and_render(upload: users.Upload):
else:
proc = None
return proc, is_stale
def _render(upload: users.Upload, proc: UploadProc, is_stale: bool) -> dict:
data = {
'name': upload.name,
'upload_id': upload.upload_id,
......@@ -73,6 +74,15 @@ def _update_and_render(upload: users.Upload):
return {key: value for key, value in data.items() if value is not None}
def _update_and_render(upload: users.Upload) -> dict:
"""
If the given upload as a processing state attached, it will attempt to update this
state and store the results, before the upload is rendered for the client.
"""
proc, is_stale = _updated_proc(upload)
return _render(upload, proc, is_stale)
class Uploads(Resource):
def get(self):
......@@ -106,6 +116,40 @@ class Upload(Resource):
return _update_and_render(upload), 200
def delete(self, upload_id):
try:
upload = users.Upload.objects(id=upload_id).first()
except mongoengine.errors.ValidationError:
print('###')
abort(400, message='%s is not a valid upload id.' % upload_id)
if upload is None:
abort(404, message='Upload with id %s does not exist.' % upload_id)
proc, is_stale = _updated_proc(upload)
if not (proc.ready() or is_stale or proc.current_task_name == 'uploading'):
abort(400, message='%s has not finished processing.' % upload_id)
logger = get_logger(__name__, upload_id=upload_id)
with logger.lnr_error('Delete upload file'):
try:
files.Upload(upload.upload_id).delete()
except KeyError:
logger.error('Upload exist, but file does not exist.')
if proc.upload_hash is not None:
with logger.lnr_error('Deleting archives.'):
files.delete_archives(proc.upload_hash)
with logger.lnr_error('Deleting indexed calcs.'):
for obj in search.Calc.search_objs(upload_hash=proc.upload_hash):
obj.delete()
with logger.lnr_error('Deleting user upload.'):
upload.delete()
return _render(upload, proc, is_stale), 200
class RepoCalc(Resource):
@staticmethod
......
......@@ -268,6 +268,13 @@ class Upload():
""" Returns the tmp directory relative version of a filename. """
return os.path.join(self.upload_extract_dir, filename)
def delete(self):
""" Delete the file from the store. Must not be open. """
try:
_client.remove_object(config.files.uploads_bucket, self.upload_id)
except minio.error.NoSuchKey:
raise KeyError(self.upload_id)
@contextmanager
def write_archive_json(archive_id) -> Generator[TextIO, None, None]:
......@@ -317,3 +324,13 @@ def open_archive_json(archive_id) -> IO:
return _client.get_object(config.files.archive_bucket, archive_id)
except minio.error.NoSuchKey:
raise KeyError()
def delete_archives(upload_hash: str):
""" Deletes all archive files for this upload_hash. """
bucket = config.files.archive_bucket
prefix = '%s/' % upload_hash
to_remove = [obj.object_name for obj in _client.list_objects(bucket, prefix)]
for _ in _client.remove_objects(bucket, to_remove):
# TODO handle potential errors
pass
......@@ -71,6 +71,12 @@ class Calc(Document):
def search(body):
return client.search(index=config.elastic.calc_index, body=body)
@staticmethod
def search_objs(**kwargs):
return Search(using=client, index=config.elastic.calc_index) \
.query('match', **kwargs) \
.execute()
@staticmethod
def upload_exists(upload_hash):
""" Returns true if there are already calcs from the given upload. """
......
......@@ -119,6 +119,19 @@ def test_create_upload_with_name(client):
assert upload['name'] == 'test_name'
def test_delete_empty_upload(client):
rv = client.post('/uploads')
assert rv.status_code == 200
upload_id = assert_upload(rv.data)['upload_id']
rv = client.delete('/uploads/%s' % upload_id)
assert rv.status_code == 200
rv = client.get('/uploads/%s' % upload_id)
assert rv.status_code == 404
@pytest.mark.parametrize("file", example_files)
@pytest.mark.timeout(30)
def test_upload_to_upload(client, file):
......
......@@ -116,6 +116,18 @@ def test_upload(uploaded_id: str):
break
def test_delete_upload(uploaded_id: str):
files.Upload(uploaded_id).delete()
try:
files.Upload(uploaded_id)
assert False
except KeyError:
pass
else:
assert False
@pytest.mark.timeout(10)
def test_upload_notification(upload_id):
@files.upload_put_handler
......@@ -162,3 +174,14 @@ def test_archive(archive_id: str):
assert 'test' in result
assert result['test'] == 'value'
def test_delete_archives(archive_id: str):
files.delete_archives(archive_id.split('/')[0])
try:
files.archive_url(archive_id)
assert False
except KeyError:
pass
else:
assert False
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment