Added more tests and test cleanup.

nomad/config.py

@@ -20,7 +20,7 @@ This module is used to store all configuration values. It makes use of
 import os
 from collections import namedtuple
 
-S3Config = namedtuple('S3', ['uploads_bucket', 'repository_bucket', 'archive_bucket'])
+FilesConfig = namedtuple('S3', ['uploads_bucket', 'repository_bucket', 'archive_bucket'])
 """ API independent configuration for the object storage. """
 
 CeleryConfig = namedtuple('Celery', [
@@ -36,7 +36,7 @@ FSConfig = namedtuple('FSConfig', ['tmp'])
 LogstashConfig = namedtuple('LogstashConfig', ['enabled', 'host', 'tcp_port'])
 """ Used to configure and enable/disable the ELK based centralized logging. """
 
-s3 = S3Config(
+files = FilesConfig(
     uploads_bucket='uploads',
     repository_bucket='repository',
     archive_bucket='archive'

nomad/files.py

@@ -74,8 +74,8 @@ if _client is None and 'sphinx' not in sys.modules:
         except minio.error.BucketAlreadyOwnedByYou:
             pass
 
-    ensure_bucket(config.s3.uploads_bucket)
-    ensure_bucket(config.s3.archive_bucket)
+    ensure_bucket(config.files.uploads_bucket)
+    ensure_bucket(config.files.archive_bucket)
 
 
 def get_presigned_upload_url(upload_id: str) -> str:
@@ -91,7 +91,7 @@ def get_presigned_upload_url(upload_id: str) -> str:
     Returns:
         The presigned URL string.
     """
-    return _client.presigned_put_object(config.s3.uploads_bucket, upload_id)
+    return _client.presigned_put_object(config.files.uploads_bucket, upload_id)
 
 
 def create_curl_upload_cmd(presigned_url: str, file_dummy: str='<ZIPFILE>') -> str:
@@ -135,7 +135,7 @@ def upload_put_handler(func: Callable[[str], None]) -> Callable[[], None]:
     def wrapper(*args, **kwargs) -> None:
         logger.info('Start listening to uploads notifications.')
 
-        events = _client.listen_bucket_notification(config.s3.uploads_bucket)
+        events = _client.listen_bucket_notification(config.files.uploads_bucket)
 
         upload_ids = upload_notifications(events)
         for upload_id in upload_ids:
@@ -182,7 +182,7 @@ class Upload():
         self.filelist: List[str] = None
 
         try:
-            self.metadata = _client.stat_object(config.s3.uploads_bucket, upload_id).metadata
+            self.metadata = _client.stat_object(config.files.uploads_bucket, upload_id).metadata
         except minio.error.NoSuchKey:
             raise KeyError(self.upload_id)
 
@@ -220,7 +220,7 @@ class Upload():
             KeyError: If the upload does not exist.
         """
         try:
-            _client.fget_object(config.s3.uploads_bucket, self.upload_id, self.upload_file)
+            _client.fget_object(config.files.uploads_bucket, self.upload_id, self.upload_file)
         except minio.error.NoSuchKey:
             raise KeyError(self.upload_id)
 
@@ -281,7 +281,7 @@ def write_archive_json(archive_id) -> Generator[TextIO, None, None]:
         length = len(binary_out.getvalue())
 
         _client.put_object(
-            config.s3.archive_bucket, archive_id, binary_out, length=length,
+            config.files.archive_bucket, archive_id, binary_out, length=length,
             content_type='application/json',
             metadata={'Content-Encoding': 'gzip'})
 
@@ -293,4 +293,4 @@ def open_archive_json(archive_id) -> IO:
     """ Returns a file-like to read the archive json. """
     # The result already is a file-like and due to the Content-Encoding metadata is
     # will automatically be un-gzipped.
-    return _client.get_object(config.s3.archive_bucket, archive_id)
+    return _client.get_object(config.files.archive_bucket, archive_id)

tests/test_files.py

@@ -31,10 +31,10 @@ example_file = './data/examples_vasp.zip'
 def uploaded_id() -> Generator[str, None, None]:
     example_upload_id = '__test_upload_id'
 
-    files._client.fput_object(config.s3.uploads_bucket, example_upload_id, example_file)
+    files._client.fput_object(config.files.uploads_bucket, example_upload_id, example_file)
     yield example_upload_id
     try:
-        files._client.remove_object(config.s3.uploads_bucket, example_upload_id)
+        files._client.remove_object(config.files.uploads_bucket, example_upload_id)
     except ResponseError:
         pass
 
@@ -44,7 +44,7 @@ def upload_id() -> Generator[str, None, None]:
     example_upload_id = '__test_upload_id'
     yield example_upload_id
     try:
-        files._client.remove_object(config.s3.uploads_bucket, example_upload_id)
+        files._client.remove_object(config.files.uploads_bucket, example_upload_id)
     except ResponseError:
         pass
 
@@ -53,10 +53,10 @@ def upload_id() -> Generator[str, None, None]:
 def archive_id() -> Generator[str, None, None]:
     example_archive_id = '__test_archive_id'
     yield example_archive_id
-    # try:
-    #     files._client.remove_object(config.s3.archive_bucket, example_archive_id)
-    # except ResponseError:
-    #     pass
+    try:
+        files._client.remove_object(config.files.archive_bucket, example_archive_id)
+    except ResponseError:
+        pass
 
 
 def test_presigned_url(upload_id):
@@ -68,7 +68,7 @@ def test_presigned_url(upload_id):
     cmd = files.create_curl_upload_cmd(upload_url).replace('<ZIPFILE>', example_file)
     subprocess.call(shlex.split(cmd))
 
-    stat = files._client.stat_object(config.s3.uploads_bucket, upload_id)
+    stat = files._client.stat_object(config.files.uploads_bucket, upload_id)
     assert stat.content_type.startswith('application/octet-steam')
 
 

tests/test_processing.py

@@ -15,9 +15,10 @@
 from typing import Generator
 import pytest
 from minio import ResponseError
+from minio.error import NoSuchBucket
 
-import nomad.files as files
 import nomad.config as config
+import nomad.files as files
 from nomad.processing import UploadProcessing
 
 example_files = ['data/examples_vasp.zip', 'data/empty.zip']
@@ -27,14 +28,23 @@ example_files = ['data/examples_vasp.zip', 'data/empty.zip']
 def uploaded_id(request) -> Generator[str, None, None]:
     example_file = request.param
     example_upload_id = example_file.replace('.zip', '')
-    files._client.fput_object(config.s3.uploads_bucket, example_upload_id, example_file)
+    files._client.fput_object(config.files.uploads_bucket, example_upload_id, example_file)
 
     yield example_upload_id
 
     try:
-        files._client.remove_object(config.s3.uploads_bucket, example_upload_id)
+        files._client.remove_object(config.files.uploads_bucket, example_upload_id)
+
+        # remove all the created archive files
+        archive_objs = files._client.list_objects(config.files.archive_bucket, recursive=True)
+        errors = files._client.remove_objects(config.files.archive_bucket, [obj.object_name for obj in archive_objs])
+        # you have to walk the iterator for minio to work (?!)
+        for _ in errors:
+            pass
     except ResponseError:
         pass
+    except NoSuchBucket:
+        pass
 
 
 def test_processing(uploaded_id):