Commit 024f52a4 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added auxfile cutoff.

parent 8f096ccb
......@@ -141,3 +141,9 @@ mail = MailConfig(
console_log_level = get_loglevel_from_env('NOMAD_CONSOLE_LOGLEVEL', default_level=logging.WARNING)
service = os.environ.get('NOMAD_SERVICE', 'unknown nomad service')
release = os.environ.get('NOMAD_RELEASE', 'devel')
auxfile_cutoff = 30
"""
Number of max auxfiles. More auxfiles means no auxfiles, but probably a directory of many
mainfiles.
"""
......@@ -570,8 +570,16 @@ class StagingUploadFiles(UploadFiles):
mainfile_basename = os.path.basename(mainfile)
calc_dir = os.path.dirname(mainfile_object.os_path)
calc_relative_dir = calc_dir[len(self._raw_dir.os_path) + 1:]
ls = os.listdir(calc_dir)
if len(ls) > config.auxfile_cutoff:
# If there are two many of them, its probably just a directory with lots of
# calculations. In this case it does not make any sense to provide thousands of
# aux files.
return [mainfile] if with_mainfile else []
aux_files = sorted(
os.path.join(calc_relative_dir, path) for path in os.listdir(calc_dir)
os.path.join(calc_relative_dir, path) for path in ls
if os.path.isfile(os.path.join(calc_dir, path)) and path != mainfile_basename)
if with_mainfile:
return [mainfile] + aux_files
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment