Commit 3f7cb9cc authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added a start_pid and only output uploads option to the migration missing cli command. [skip ci]

parent 7f620adf
Pipeline #53212 skipped
......@@ -284,9 +284,21 @@ def upload(
@migration.command(help='Get an report about not migrated calcs. This connects directly to nomad dbs, like admin commands.')
def missing():
@click.option('--start-pid', type=int, default=0, help='Only index calculations with PID greater equal the given value')
@click.option('--uploads', is_flag=True, help='Instead of extensive information, only provide a list of missing uploads')
def missing(start_pid, uploads):
infrastructure.setup_logging()
infrastructure.setup_mongo()
report = missing_calcs_data()
print(json.dumps(report, indent=2))
report = missing_calcs_data(start_pid=start_pid)
if not uploads:
print(json.dumps(report, indent=2))
else:
uploads = set()
for value in report.values():
upload = value.get('source_upload_id', None)
if upload is not None:
uploads.append(upload)
for upload in uploads:
print(upload)
......@@ -128,7 +128,7 @@ def create_package_zip(
logger.info('package zip created')
def missing_calcs_data():
def missing_calcs_data(start_pid: int = 0):
""" Produces data about missing calculations """
results = utils.POPO(
no_package=[],
......@@ -222,7 +222,7 @@ def missing_calcs_data():
# aggregate missing calcs based on uploads
source_uploads = SourceCalc._get_collection().aggregate([
{'$match': {'migration_version': -1, 'upload': {'$nin': not_check_uploads}}},
{'$match': {'migration_version': -1, '_id': {'$gte': start_pid}, 'upload': {'$nin': not_check_uploads}}},
{'$group': {'_id': '$upload', 'calcs': {'$push': {'mainfile': '$mainfile', 'pid': '$metadata.pid'}}}}])
source_uploads = list(source_uploads)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment