Commit 6643407e authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Optimized buffers in package from tar command. [skip ci]

parent 0b5dccaa
Pipeline #48084 skipped
...@@ -202,6 +202,8 @@ def clean_es(dry): ...@@ -202,6 +202,8 @@ def clean_es(dry):
search.aggs.bucket('uploads', A('terms', field='upload_id', size=12000)) search.aggs.bucket('uploads', A('terms', field='upload_id', size=12000))
response = search.execute() response = search.execute()
print('Found %d uploads in ES.' % len(response.aggregations.uploads.buckets))
for bucket in response.aggregations.uploads.buckets: for bucket in response.aggregations.uploads.buckets:
yield bucket.key, bucket.doc_count yield bucket.key, bucket.doc_count
......
...@@ -304,7 +304,8 @@ class Package(Document): ...@@ -304,7 +304,8 @@ class Package(Document):
""" """
logger = utils.get_logger(__name__) logger = utils.get_logger(__name__)
f = io.open(source_tar_path, 'rb', buffering=2100000000) f = io.open(source_tar_path, 'rb', buffering=128 * 1024 * 1024)
tf = tarfile.TarFile.open(fileobj=f, copybufsize=1024 * 1024) # type: ignore tf = tarfile.TarFile.open(fileobj=f, copybufsize=1024 * 1024) # type: ignore
if offset is not None: if offset is not None:
tf.offset = offset # type: ignore tf.offset = offset # type: ignore
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment