Commit 9dbb3bda authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Optional compression on archive.

parent 7dcea1ce
...@@ -20,7 +20,8 @@ This module is used to store all configuration values. It makes use of ...@@ -20,7 +20,8 @@ This module is used to store all configuration values. It makes use of
import os import os
from collections import namedtuple from collections import namedtuple
FilesConfig = namedtuple('S3', ['uploads_bucket', 'repository_bucket', 'archive_bucket']) FilesConfig = namedtuple(
'FilesConfig', ['uploads_bucket', 'repository_bucket', 'archive_bucket', 'compress_archive'])
""" API independent configuration for the object storage. """ """ API independent configuration for the object storage. """
CeleryConfig = namedtuple('Celery', [ CeleryConfig = namedtuple('Celery', [
...@@ -39,7 +40,8 @@ LogstashConfig = namedtuple('LogstashConfig', ['enabled', 'host', 'tcp_port']) ...@@ -39,7 +40,8 @@ LogstashConfig = namedtuple('LogstashConfig', ['enabled', 'host', 'tcp_port'])
files = FilesConfig( files = FilesConfig(
uploads_bucket='uploads', uploads_bucket='uploads',
repository_bucket='repository', repository_bucket='repository',
archive_bucket='archive' archive_bucket='archive',
compress_archive=False
) )
celery = CeleryConfig( celery = CeleryConfig(
rabbit_host=os.environ.get('NOMAD_RABBITMQ_HOST', 'localhost'), rabbit_host=os.environ.get('NOMAD_RABBITMQ_HOST', 'localhost'),
......
...@@ -271,21 +271,28 @@ class Upload(): ...@@ -271,21 +271,28 @@ class Upload():
def write_archive_json(archive_id) -> Generator[TextIO, None, None]: def write_archive_json(archive_id) -> Generator[TextIO, None, None]:
""" Context manager that yiels a file-like to write the archive json. """ """ Context manager that yiels a file-like to write the archive json. """
binary_out = io.BytesIO() binary_out = io.BytesIO()
gzip_wrapper = cast(TextIO, gzip.open(binary_out, 'wt')) if config.files.compress_archive:
gzip_wrapper = cast(TextIO, gzip.open(binary_out, 'wt'))
out = gzip_wrapper
metadata = {'Content-Encoding': 'gzip'}
else:
text_wrapper = io.TextIOWrapper(binary_out, encoding='utf-8')
out = text_wrapper
metadata = {}
try: try:
yield gzip_wrapper yield out
finally: finally:
gzip_wrapper.flush() out.flush()
binary_out.seek(0) binary_out.seek(0)
length = len(binary_out.getvalue()) length = len(binary_out.getvalue())
_client.put_object( _client.put_object(
config.files.archive_bucket, archive_id, binary_out, length=length, config.files.archive_bucket, archive_id, binary_out, length=length,
content_type='application/json', content_type='application/json',
metadata={'Content-Encoding': 'gzip'}) metadata=metadata)
gzip_wrapper.close() out.close()
binary_out.close() binary_out.close()
......
...@@ -33,6 +33,7 @@ def uploaded_id(request) -> Generator[str, None, None]: ...@@ -33,6 +33,7 @@ def uploaded_id(request) -> Generator[str, None, None]:
yield example_upload_id yield example_upload_id
try: try:
# remove the created uploads
files._client.remove_object(config.files.uploads_bucket, example_upload_id) files._client.remove_object(config.files.uploads_bucket, example_upload_id)
# remove all the created archive files # remove all the created archive files
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment