diff --git a/nomad/api.py b/nomad/api.py
index 5893de3006f39866d86803fcd89f2ec558cc2f16..6eedda86d749f00767e77dc2e46ede3e1b33c43f 100644
--- a/nomad/api.py
+++ b/nomad/api.py
@@ -69,8 +69,25 @@ class Upload(Resource):
return Uploads._render(upload), 200
+class Calc(Resource):
+ def get(self, upload_hash, calc_hash):
+ archive_id = '%s/%s' % (upload_hash, calc_hash)
+ logger = get_logger(__name__, archive_id=archive_id)
+ try:
+ file = files.open_archive_json(archive_id)
+ return file, 200
+ except KeyError:
+ abort(404, message='Archive %s does not exist.' % archive_id)
+ except Exception as e:
+ logger.error('Exception on reading archive', exc_info=e)
+ abort(500, message='Could not read the archive.')
+ finally:
+ file.close()
+
+
api.add_resource(Uploads, '/uploads')
api.add_resource(Upload, '/uploads/<string:upload_id>')
+api.add_resource(Calc, '/archive/<string:upload_hash>/<string:calc_hash>')
def start_upload_handler(quit=False):
diff --git a/nomad/files.py b/nomad/files.py
index d5d3fe59cb99fb163671af82e35c0779dd12c5a0..0b76806e6432233aef90e40548d32058ea4a2e5d 100644
--- a/nomad/files.py
+++ b/nomad/files.py
@@ -41,7 +41,7 @@ Uploads
from typing import Callable, List, Any, Generator, IO, TextIO, cast
import sys
import os
-from os.path import join
+import os.path
from zipfile import ZipFile, BadZipFile
import shutil
from minio import Minio
@@ -176,8 +176,8 @@ class Upload():
"""
def __init__(self, upload_id: str) -> None:
self.upload_id = upload_id
- self.upload_file: str = '%s/uploads/%s.zip' % (config.fs.tmp, upload_id)
- self.upload_extract_dir: str = '%s/uploads_extracted/%s' % (config.fs.tmp, upload_id)
+ self.upload_file: str = os.path.join(config.fs.tmp, 'uploads', upload_id)
+ self.upload_extract_dir: str = os.path.join(config.fs.tmp, 'uploads_extracted', upload_id)
self.filelist: List[str] = None
try:
@@ -218,6 +218,9 @@ class Upload():
UploadError: If some IO went wrong.
KeyError: If the upload does not exist.
"""
+ os.makedirs(os.path.join(config.fs.tmp, 'uploads'), exist_ok=True)
+ os.makedirs(os.path.join(config.fs.tmp, 'uploads_extracted'), exist_ok=True)
+
try:
_client.fget_object(config.files.uploads_bucket, self.upload_id, self.upload_file)
except minio.error.NoSuchKey:
@@ -263,7 +266,7 @@ class Upload():
def get_path(self, filename: str) -> str:
""" Returns the tmp directory relative version of a filename. """
- return join(self.upload_extract_dir, filename)
+ return os.path.join(self.upload_extract_dir, filename)
@contextmanager
diff --git a/tests/test_api.py b/tests/test_api.py
index 543f5fa98faa3df89a6120667152bcc0a807a0a9..01ceed9e5312f42e66ff04bfe69479f64247ee2b 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -6,11 +6,13 @@ import time
import json
from mongoengine import connect
from mongoengine.connection import disconnect
-from minio.error import ResponseError
from nomad import config, api, files
from tests.test_processing import example_files
+from tests.test_files import assert_exists
+# import fixtures
+from tests.test_files import clear_files # pylint: disable=unused-import
@pytest.fixture
@@ -101,10 +103,7 @@ def test_upload_to_upload(client, file):
handle_uploads_thread.join()
- try:
- files._client.remove_object(config.files.uploads_bucket, upload['id'])
- except ResponseError:
- assert False
+ assert_exists(config.files.uploads_bucket, upload['id'])
@pytest.mark.parametrize("file", example_files)
@@ -136,8 +135,4 @@ def test_processing(client, file):
break
assert upload['processing']['status'] == 'SUCCESS'
-
- try:
- files._client.remove_object(config.files.uploads_bucket, upload['id'])
- except ResponseError:
- assert False
+ assert_exists(config.files.uploads_bucket, upload['id'])
diff --git a/tests/test_files.py b/tests/test_files.py
index 627a2db3a4bbf660d253d086c7d35a90fc3b94a6..06baf3dd9740b6fb59e28bd9c344a2097774ee88 100644
--- a/tests/test_files.py
+++ b/tests/test_files.py
@@ -13,13 +13,16 @@
# limitations under the License.
import pytest
-from minio import ResponseError
from threading import Thread
import subprocess
import shlex
import time
from typing import Generator
import json
+import shutil
+import os.path
+from minio import ResponseError
+from minio.error import NoSuchBucket
import nomad.files as files
import nomad.config as config
@@ -28,36 +31,54 @@ example_file = 'data/examples_vasp.zip'
empty_file = 'data/empty.zip'
-@pytest.fixture
-def uploaded_id() -> Generator[str, None, None]:
+def assert_exists(bucket_name, object_name):
+ try:
+ stats = files._client.stat_object(bucket_name, object_name)
+ assert stats is not None
+ except NoSuchBucket:
+ assert False
+ except ResponseError:
+ assert False
+
+
+@pytest.fixture(scope='function')
+def clear_files():
+ """ Utility fixture that removes all files from files and tmp after test. """
+ try:
+ yield
+ finally:
+ try:
+ for bucket in [config.files.uploads_bucket, config.files.archive_bucket]:
+ to_remove = [obj.object_name for obj in files._client.list_objects(bucket)]
+ for _ in files._client.remove_objects(bucket, to_remove):
+ pass
+ except NoSuchBucket:
+ pass
+ except ResponseError:
+ pass
+
+ shutil.rmtree(os.path.join(config.fs.tmp, 'uploads'), ignore_errors=True)
+ shutil.rmtree(os.path.join(config.fs.tmp, 'uploads_extracted'), ignore_errors=True)
+
+
+@pytest.fixture(scope='function')
+def uploaded_id(clear_files) -> Generator[str, None, None]:
example_upload_id = '__test_upload_id'
files._client.fput_object(config.files.uploads_bucket, example_upload_id, example_file)
yield example_upload_id
- try:
- files._client.remove_object(config.files.uploads_bucket, example_upload_id)
- except ResponseError:
- pass
-@pytest.fixture
-def upload_id() -> Generator[str, None, None]:
+@pytest.fixture(scope='function')
+def upload_id(clear_files) -> Generator[str, None, None]:
example_upload_id = '__test_upload_id'
yield example_upload_id
- try:
- files._client.remove_object(config.files.uploads_bucket, example_upload_id)
- except ResponseError:
- pass
-@pytest.fixture
-def archive_id() -> Generator[str, None, None]:
+@pytest.fixture(scope='function')
+def archive_id(clear_files) -> Generator[str, None, None]:
example_archive_id = '__test_archive_id'
yield example_archive_id
- try:
- files._client.remove_object(config.files.archive_bucket, example_archive_id)
- except ResponseError:
- pass
def test_presigned_url(upload_id):
diff --git a/tests/test_processing.py b/tests/test_processing.py
index 3404e2b002f9fcce23c07d91c728e4e0cf6693d7..edf586e35d07dc546ee169cf6d2dc554faf769c1 100644
--- a/tests/test_processing.py
+++ b/tests/test_processing.py
@@ -20,16 +20,15 @@ reading from the redis result backend, even though all task apperently ended suc
from typing import Generator
import pytest
-from minio import ResponseError
-from minio.error import NoSuchBucket
import nomad.config as config
import nomad.files as files
from nomad.processing import UploadProcessing
-# delete search index after each test via imported fixture
-from tests.test_search import index # pylint: disable=unused-import
from tests.test_files import example_file, empty_file
+# import fixtures
+from tests.test_search import index # pylint: disable=unused-import
+from tests.test_files import clear_files # pylint: disable=unused-import
example_files = [empty_file, example_file]
@@ -58,28 +57,13 @@ def celery_config():
@pytest.fixture(scope='function', params=example_files)
-def uploaded_id(request) -> Generator[str, None, None]:
+def uploaded_id(request, clear_files) -> Generator[str, None, None]:
example_file = request.param
example_upload_id = example_file.replace('.zip', '')
files._client.fput_object(config.files.uploads_bucket, example_upload_id, example_file)
yield example_upload_id
- try:
- # remove the created uploads
- files._client.remove_object(config.files.uploads_bucket, example_upload_id)
-
- # remove all the created archive files
- archive_objs = files._client.list_objects(config.files.archive_bucket, recursive=True)
- errors = files._client.remove_objects(config.files.archive_bucket, [obj.object_name for obj in archive_objs])
- # you have to walk the iterator for minio to work (?!)
- for _ in errors:
- pass
- except ResponseError:
- pass
- except NoSuchBucket:
- pass
-
def test_processing(uploaded_id, celery_session_worker):
run = UploadProcessing(uploaded_id)