Commit 16cbf234 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Improved test cleanup through more fixtures.

parent 21a451f4
......@@ -69,8 +69,25 @@ class Upload(Resource):
return Uploads._render(upload), 200
class Calc(Resource):
def get(self, upload_hash, calc_hash):
archive_id = '%s/%s' % (upload_hash, calc_hash)
logger = get_logger(__name__, archive_id=archive_id)
try:
file = files.open_archive_json(archive_id)
return file, 200
except KeyError:
abort(404, message='Archive %s does not exist.' % archive_id)
except Exception as e:
logger.error('Exception on reading archive', exc_info=e)
abort(500, message='Could not read the archive.')
finally:
file.close()
api.add_resource(Uploads, '/uploads')
api.add_resource(Upload, '/uploads/<string:upload_id>')
api.add_resource(Calc, '/archive/<string:upload_hash>/<string:calc_hash>')
def start_upload_handler(quit=False):
......
......@@ -41,7 +41,7 @@ Uploads
from typing import Callable, List, Any, Generator, IO, TextIO, cast
import sys
import os
from os.path import join
import os.path
from zipfile import ZipFile, BadZipFile
import shutil
from minio import Minio
......@@ -176,8 +176,8 @@ class Upload():
"""
def __init__(self, upload_id: str) -> None:
self.upload_id = upload_id
self.upload_file: str = '%s/uploads/%s.zip' % (config.fs.tmp, upload_id)
self.upload_extract_dir: str = '%s/uploads_extracted/%s' % (config.fs.tmp, upload_id)
self.upload_file: str = os.path.join(config.fs.tmp, 'uploads', upload_id)
self.upload_extract_dir: str = os.path.join(config.fs.tmp, 'uploads_extracted', upload_id)
self.filelist: List[str] = None
try:
......@@ -218,6 +218,9 @@ class Upload():
UploadError: If some IO went wrong.
KeyError: If the upload does not exist.
"""
os.makedirs(os.path.join(config.fs.tmp, 'uploads'), exist_ok=True)
os.makedirs(os.path.join(config.fs.tmp, 'uploads_extracted'), exist_ok=True)
try:
_client.fget_object(config.files.uploads_bucket, self.upload_id, self.upload_file)
except minio.error.NoSuchKey:
......@@ -263,7 +266,7 @@ class Upload():
def get_path(self, filename: str) -> str:
""" Returns the tmp directory relative version of a filename. """
return join(self.upload_extract_dir, filename)
return os.path.join(self.upload_extract_dir, filename)
@contextmanager
......
......@@ -6,11 +6,13 @@ import time
import json
from mongoengine import connect
from mongoengine.connection import disconnect
from minio.error import ResponseError
from nomad import config, api, files
from tests.test_processing import example_files
from tests.test_files import assert_exists
# import fixtures
from tests.test_files import clear_files # pylint: disable=unused-import
@pytest.fixture
......@@ -101,10 +103,7 @@ def test_upload_to_upload(client, file):
handle_uploads_thread.join()
try:
files._client.remove_object(config.files.uploads_bucket, upload['id'])
except ResponseError:
assert False
assert_exists(config.files.uploads_bucket, upload['id'])
@pytest.mark.parametrize("file", example_files)
......@@ -136,8 +135,4 @@ def test_processing(client, file):
break
assert upload['processing']['status'] == 'SUCCESS'
try:
files._client.remove_object(config.files.uploads_bucket, upload['id'])
except ResponseError:
assert False
assert_exists(config.files.uploads_bucket, upload['id'])
......@@ -13,13 +13,16 @@
# limitations under the License.
import pytest
from minio import ResponseError
from threading import Thread
import subprocess
import shlex
import time
from typing import Generator
import json
import shutil
import os.path
from minio import ResponseError
from minio.error import NoSuchBucket
import nomad.files as files
import nomad.config as config
......@@ -28,36 +31,54 @@ example_file = 'data/examples_vasp.zip'
empty_file = 'data/empty.zip'
@pytest.fixture
def uploaded_id() -> Generator[str, None, None]:
def assert_exists(bucket_name, object_name):
try:
stats = files._client.stat_object(bucket_name, object_name)
assert stats is not None
except NoSuchBucket:
assert False
except ResponseError:
assert False
@pytest.fixture(scope='function')
def clear_files():
""" Utility fixture that removes all files from files and tmp after test. """
try:
yield
finally:
try:
for bucket in [config.files.uploads_bucket, config.files.archive_bucket]:
to_remove = [obj.object_name for obj in files._client.list_objects(bucket)]
for _ in files._client.remove_objects(bucket, to_remove):
pass
except NoSuchBucket:
pass
except ResponseError:
pass
shutil.rmtree(os.path.join(config.fs.tmp, 'uploads'), ignore_errors=True)
shutil.rmtree(os.path.join(config.fs.tmp, 'uploads_extracted'), ignore_errors=True)
@pytest.fixture(scope='function')
def uploaded_id(clear_files) -> Generator[str, None, None]:
example_upload_id = '__test_upload_id'
files._client.fput_object(config.files.uploads_bucket, example_upload_id, example_file)
yield example_upload_id
try:
files._client.remove_object(config.files.uploads_bucket, example_upload_id)
except ResponseError:
pass
@pytest.fixture
def upload_id() -> Generator[str, None, None]:
@pytest.fixture(scope='function')
def upload_id(clear_files) -> Generator[str, None, None]:
example_upload_id = '__test_upload_id'
yield example_upload_id
try:
files._client.remove_object(config.files.uploads_bucket, example_upload_id)
except ResponseError:
pass
@pytest.fixture
def archive_id() -> Generator[str, None, None]:
@pytest.fixture(scope='function')
def archive_id(clear_files) -> Generator[str, None, None]:
example_archive_id = '__test_archive_id'
yield example_archive_id
try:
files._client.remove_object(config.files.archive_bucket, example_archive_id)
except ResponseError:
pass
def test_presigned_url(upload_id):
......
......@@ -20,16 +20,15 @@ reading from the redis result backend, even though all task apperently ended suc
from typing import Generator
import pytest
from minio import ResponseError
from minio.error import NoSuchBucket
import nomad.config as config
import nomad.files as files
from nomad.processing import UploadProcessing
# delete search index after each test via imported fixture
from tests.test_search import index # pylint: disable=unused-import
from tests.test_files import example_file, empty_file
# import fixtures
from tests.test_search import index # pylint: disable=unused-import
from tests.test_files import clear_files # pylint: disable=unused-import
example_files = [empty_file, example_file]
......@@ -58,28 +57,13 @@ def celery_config():
@pytest.fixture(scope='function', params=example_files)
def uploaded_id(request) -> Generator[str, None, None]:
def uploaded_id(request, clear_files) -> Generator[str, None, None]:
example_file = request.param
example_upload_id = example_file.replace('.zip', '')
files._client.fput_object(config.files.uploads_bucket, example_upload_id, example_file)
yield example_upload_id
try:
# remove the created uploads
files._client.remove_object(config.files.uploads_bucket, example_upload_id)
# remove all the created archive files
archive_objs = files._client.list_objects(config.files.archive_bucket, recursive=True)
errors = files._client.remove_objects(config.files.archive_bucket, [obj.object_name for obj in archive_objs])
# you have to walk the iterator for minio to work (?!)
for _ in errors:
pass
except ResponseError:
pass
except NoSuchBucket:
pass
def test_processing(uploaded_id, celery_session_worker):
run = UploadProcessing(uploaded_id)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment