Commit 9dbb3bda authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Optional compression on archive.

parent 7dcea1ce
......@@ -20,7 +20,8 @@ This module is used to store all configuration values. It makes use of
import os
from collections import namedtuple
FilesConfig = namedtuple('S3', ['uploads_bucket', 'repository_bucket', 'archive_bucket'])
FilesConfig = namedtuple(
'FilesConfig', ['uploads_bucket', 'repository_bucket', 'archive_bucket', 'compress_archive'])
""" API independent configuration for the object storage. """
CeleryConfig = namedtuple('Celery', [
......@@ -39,7 +40,8 @@ LogstashConfig = namedtuple('LogstashConfig', ['enabled', 'host', 'tcp_port'])
files = FilesConfig(
uploads_bucket='uploads',
repository_bucket='repository',
archive_bucket='archive'
archive_bucket='archive',
compress_archive=False
)
celery = CeleryConfig(
rabbit_host=os.environ.get('NOMAD_RABBITMQ_HOST', 'localhost'),
......
......@@ -271,21 +271,28 @@ class Upload():
def write_archive_json(archive_id) -> Generator[TextIO, None, None]:
""" Context manager that yiels a file-like to write the archive json. """
binary_out = io.BytesIO()
if config.files.compress_archive:
gzip_wrapper = cast(TextIO, gzip.open(binary_out, 'wt'))
out = gzip_wrapper
metadata = {'Content-Encoding': 'gzip'}
else:
text_wrapper = io.TextIOWrapper(binary_out, encoding='utf-8')
out = text_wrapper
metadata = {}
try:
yield gzip_wrapper
yield out
finally:
gzip_wrapper.flush()
out.flush()
binary_out.seek(0)
length = len(binary_out.getvalue())
_client.put_object(
config.files.archive_bucket, archive_id, binary_out, length=length,
content_type='application/json',
metadata={'Content-Encoding': 'gzip'})
metadata=metadata)
gzip_wrapper.close()
out.close()
binary_out.close()
......
......@@ -33,6 +33,7 @@ def uploaded_id(request) -> Generator[str, None, None]:
yield example_upload_id
try:
# remove the created uploads
files._client.remove_object(config.files.uploads_bucket, example_upload_id)
# remove all the created archive files
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment