Skip to content
Snippets Groups Projects
Commit 024f52a4 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added auxfile cutoff.

parent 8f096ccb
Branches
Tags
1 merge request!33Merge version 0.4.0
...@@ -141,3 +141,9 @@ mail = MailConfig( ...@@ -141,3 +141,9 @@ mail = MailConfig(
console_log_level = get_loglevel_from_env('NOMAD_CONSOLE_LOGLEVEL', default_level=logging.WARNING) console_log_level = get_loglevel_from_env('NOMAD_CONSOLE_LOGLEVEL', default_level=logging.WARNING)
service = os.environ.get('NOMAD_SERVICE', 'unknown nomad service') service = os.environ.get('NOMAD_SERVICE', 'unknown nomad service')
release = os.environ.get('NOMAD_RELEASE', 'devel') release = os.environ.get('NOMAD_RELEASE', 'devel')
auxfile_cutoff = 30
"""
Number of max auxfiles. More auxfiles means no auxfiles, but probably a directory of many
mainfiles.
"""
...@@ -570,8 +570,16 @@ class StagingUploadFiles(UploadFiles): ...@@ -570,8 +570,16 @@ class StagingUploadFiles(UploadFiles):
mainfile_basename = os.path.basename(mainfile) mainfile_basename = os.path.basename(mainfile)
calc_dir = os.path.dirname(mainfile_object.os_path) calc_dir = os.path.dirname(mainfile_object.os_path)
calc_relative_dir = calc_dir[len(self._raw_dir.os_path) + 1:] calc_relative_dir = calc_dir[len(self._raw_dir.os_path) + 1:]
ls = os.listdir(calc_dir)
if len(ls) > config.auxfile_cutoff:
# If there are two many of them, its probably just a directory with lots of
# calculations. In this case it does not make any sense to provide thousands of
# aux files.
return [mainfile] if with_mainfile else []
aux_files = sorted( aux_files = sorted(
os.path.join(calc_relative_dir, path) for path in os.listdir(calc_dir) os.path.join(calc_relative_dir, path) for path in ls
if os.path.isfile(os.path.join(calc_dir, path)) and path != mainfile_basename) if os.path.isfile(os.path.join(calc_dir, path)) and path != mainfile_basename)
if with_mainfile: if with_mainfile:
return [mainfile] + aux_files return [mainfile] + aux_files
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment