Optional compression on archive.

nomad/config.py

@@ -20,7 +20,8 @@ This module is used to store all configuration values. It makes use of
 import os
 from collections import namedtuple
 
-FilesConfig = namedtuple('S3', ['uploads_bucket', 'repository_bucket', 'archive_bucket'])
+FilesConfig = namedtuple(
+    'FilesConfig', ['uploads_bucket', 'repository_bucket', 'archive_bucket', 'compress_archive'])
 """ API independent configuration for the object storage. """
 
 CeleryConfig = namedtuple('Celery', [
@@ -39,7 +40,8 @@ LogstashConfig = namedtuple('LogstashConfig', ['enabled', 'host', 'tcp_port'])
 files = FilesConfig(
     uploads_bucket='uploads',
     repository_bucket='repository',
-    archive_bucket='archive'
+    archive_bucket='archive',
+    compress_archive=False
 )
 celery = CeleryConfig(
     rabbit_host=os.environ.get('NOMAD_RABBITMQ_HOST', 'localhost'),

nomad/files.py

@@ -271,21 +271,28 @@ class Upload():
 def write_archive_json(archive_id) -> Generator[TextIO, None, None]:
     """ Context manager that yiels a file-like to write the archive json. """
     binary_out = io.BytesIO()
-    gzip_wrapper = cast(TextIO, gzip.open(binary_out, 'wt'))
+    if config.files.compress_archive:
+        gzip_wrapper = cast(TextIO, gzip.open(binary_out, 'wt'))
+        out = gzip_wrapper
+        metadata = {'Content-Encoding': 'gzip'}
+    else:
+        text_wrapper = io.TextIOWrapper(binary_out, encoding='utf-8')
+        out = text_wrapper
+        metadata = {}
 
     try:
-        yield gzip_wrapper
+        yield out
     finally:
-        gzip_wrapper.flush()
+        out.flush()
         binary_out.seek(0)
         length = len(binary_out.getvalue())
 
         _client.put_object(
             config.files.archive_bucket, archive_id, binary_out, length=length,
             content_type='application/json',
-            metadata={'Content-Encoding': 'gzip'})
+            metadata=metadata)
 
-        gzip_wrapper.close()
+        out.close()
         binary_out.close()
 
 

tests/test_processing.py

@@ -33,6 +33,7 @@ def uploaded_id(request) -> Generator[str, None, None]:
     yield example_upload_id
 
     try:
+        # remove the created uploads
         files._client.remove_object(config.files.uploads_bucket, example_upload_id)
 
         # remove all the created archive files