Commit b5f20be0 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added more tests and test cleanup.

parent 57962d7f
......@@ -20,7 +20,7 @@ This module is used to store all configuration values. It makes use of
import os
from collections import namedtuple
S3Config = namedtuple('S3', ['uploads_bucket', 'repository_bucket', 'archive_bucket'])
FilesConfig = namedtuple('S3', ['uploads_bucket', 'repository_bucket', 'archive_bucket'])
""" API independent configuration for the object storage. """
CeleryConfig = namedtuple('Celery', [
......@@ -36,7 +36,7 @@ FSConfig = namedtuple('FSConfig', ['tmp'])
LogstashConfig = namedtuple('LogstashConfig', ['enabled', 'host', 'tcp_port'])
""" Used to configure and enable/disable the ELK based centralized logging. """
s3 = S3Config(
files = FilesConfig(
uploads_bucket='uploads',
repository_bucket='repository',
archive_bucket='archive'
......
......@@ -74,8 +74,8 @@ if _client is None and 'sphinx' not in sys.modules:
except minio.error.BucketAlreadyOwnedByYou:
pass
ensure_bucket(config.s3.uploads_bucket)
ensure_bucket(config.s3.archive_bucket)
ensure_bucket(config.files.uploads_bucket)
ensure_bucket(config.files.archive_bucket)
def get_presigned_upload_url(upload_id: str) -> str:
......@@ -91,7 +91,7 @@ def get_presigned_upload_url(upload_id: str) -> str:
Returns:
The presigned URL string.
"""
return _client.presigned_put_object(config.s3.uploads_bucket, upload_id)
return _client.presigned_put_object(config.files.uploads_bucket, upload_id)
def create_curl_upload_cmd(presigned_url: str, file_dummy: str='<ZIPFILE>') -> str:
......@@ -135,7 +135,7 @@ def upload_put_handler(func: Callable[[str], None]) -> Callable[[], None]:
def wrapper(*args, **kwargs) -> None:
logger.info('Start listening to uploads notifications.')
events = _client.listen_bucket_notification(config.s3.uploads_bucket)
events = _client.listen_bucket_notification(config.files.uploads_bucket)
upload_ids = upload_notifications(events)
for upload_id in upload_ids:
......@@ -182,7 +182,7 @@ class Upload():
self.filelist: List[str] = None
try:
self.metadata = _client.stat_object(config.s3.uploads_bucket, upload_id).metadata
self.metadata = _client.stat_object(config.files.uploads_bucket, upload_id).metadata
except minio.error.NoSuchKey:
raise KeyError(self.upload_id)
......@@ -220,7 +220,7 @@ class Upload():
KeyError: If the upload does not exist.
"""
try:
_client.fget_object(config.s3.uploads_bucket, self.upload_id, self.upload_file)
_client.fget_object(config.files.uploads_bucket, self.upload_id, self.upload_file)
except minio.error.NoSuchKey:
raise KeyError(self.upload_id)
......@@ -281,7 +281,7 @@ def write_archive_json(archive_id) -> Generator[TextIO, None, None]:
length = len(binary_out.getvalue())
_client.put_object(
config.s3.archive_bucket, archive_id, binary_out, length=length,
config.files.archive_bucket, archive_id, binary_out, length=length,
content_type='application/json',
metadata={'Content-Encoding': 'gzip'})
......@@ -293,4 +293,4 @@ def open_archive_json(archive_id) -> IO:
""" Returns a file-like to read the archive json. """
# The result already is a file-like and due to the Content-Encoding metadata is
# will automatically be un-gzipped.
return _client.get_object(config.s3.archive_bucket, archive_id)
return _client.get_object(config.files.archive_bucket, archive_id)
......@@ -31,10 +31,10 @@ example_file = './data/examples_vasp.zip'
def uploaded_id() -> Generator[str, None, None]:
example_upload_id = '__test_upload_id'
files._client.fput_object(config.s3.uploads_bucket, example_upload_id, example_file)
files._client.fput_object(config.files.uploads_bucket, example_upload_id, example_file)
yield example_upload_id
try:
files._client.remove_object(config.s3.uploads_bucket, example_upload_id)
files._client.remove_object(config.files.uploads_bucket, example_upload_id)
except ResponseError:
pass
......@@ -44,7 +44,7 @@ def upload_id() -> Generator[str, None, None]:
example_upload_id = '__test_upload_id'
yield example_upload_id
try:
files._client.remove_object(config.s3.uploads_bucket, example_upload_id)
files._client.remove_object(config.files.uploads_bucket, example_upload_id)
except ResponseError:
pass
......@@ -53,10 +53,10 @@ def upload_id() -> Generator[str, None, None]:
def archive_id() -> Generator[str, None, None]:
example_archive_id = '__test_archive_id'
yield example_archive_id
# try:
# files._client.remove_object(config.s3.archive_bucket, example_archive_id)
# except ResponseError:
# pass
try:
files._client.remove_object(config.files.archive_bucket, example_archive_id)
except ResponseError:
pass
def test_presigned_url(upload_id):
......@@ -68,7 +68,7 @@ def test_presigned_url(upload_id):
cmd = files.create_curl_upload_cmd(upload_url).replace('<ZIPFILE>', example_file)
subprocess.call(shlex.split(cmd))
stat = files._client.stat_object(config.s3.uploads_bucket, upload_id)
stat = files._client.stat_object(config.files.uploads_bucket, upload_id)
assert stat.content_type.startswith('application/octet-steam')
......
......@@ -15,9 +15,10 @@
from typing import Generator
import pytest
from minio import ResponseError
from minio.error import NoSuchBucket
import nomad.files as files
import nomad.config as config
import nomad.files as files
from nomad.processing import UploadProcessing
example_files = ['data/examples_vasp.zip', 'data/empty.zip']
......@@ -27,14 +28,23 @@ example_files = ['data/examples_vasp.zip', 'data/empty.zip']
def uploaded_id(request) -> Generator[str, None, None]:
example_file = request.param
example_upload_id = example_file.replace('.zip', '')
files._client.fput_object(config.s3.uploads_bucket, example_upload_id, example_file)
files._client.fput_object(config.files.uploads_bucket, example_upload_id, example_file)
yield example_upload_id
try:
files._client.remove_object(config.s3.uploads_bucket, example_upload_id)
files._client.remove_object(config.files.uploads_bucket, example_upload_id)
# remove all the created archive files
archive_objs = files._client.list_objects(config.files.archive_bucket, recursive=True)
errors = files._client.remove_objects(config.files.archive_bucket, [obj.object_name for obj in archive_objs])
# you have to walk the iterator for minio to work (?!)
for _ in errors:
pass
except ResponseError:
pass
except NoSuchBucket:
pass
def test_processing(uploaded_id):
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment