Commit ea92cccd authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Use subquery lazyloading for datasets to comfort the production repo db.

parent 13db6d28
Pipeline #42522 passed with stages
in 19 minutes and 5 seconds
......@@ -47,11 +47,15 @@ def migration(host, port, user, password, dbname):
@click.option('--per-query', default=100, help='We index many objects with one query. Default is 100.')
def index(drop, with_metadata, per_query):
start = time.time()
indexed = 0
for _, total in _migration.index(drop=drop, with_metadata=with_metadata, per_query=int(per_query)):
indexed += 1
eta = total * ((time.time() - start) / indexed)
print('indexed: %8d, total: %8d, ETA: %s\r' % (indexed, total, datetime.timedelta(seconds=eta)), end='')
indexed_total = 0
indexed_calcs = 0
for calc, total in _migration.index(drop=drop, with_metadata=with_metadata, per_query=int(per_query)):
indexed_total += 1
indexed_calcs += 1 if calc is not None else 0
eta = total * ((time.time() - start) / indexed_total)
print(
'indexed: %8d, calcs: %8d, total: %8d, ETA: %s\r' %
(indexed_total, indexed_calcs, total, datetime.timedelta(seconds=eta)), end='')
print('done')
......
......@@ -45,7 +45,7 @@ class Calc(Base, datamodel.Calc): # type: ignore
secondary=calc_dataset_containment,
primaryjoin=calc_dataset_containment.c.children_calc_id == coe_calc_id,
secondaryjoin=calc_dataset_containment.c.parent_calc_id == coe_calc_id,
backref='children', lazy='joined', join_depth=2)
backref='children', lazy='subquery', join_depth=1)
@classmethod
def load_from(cls, obj):
......
......@@ -100,7 +100,8 @@ class SourceCalc(Document):
calcs = source_query.filter(Calc.coe_calc_id > start_pid).order_by(Calc.coe_calc_id).limit(per_query)
source_calcs = []
for calc in calcs:
if calc.calc_metadata.filenames is None:
if calc.calc_metadata is None or calc.calc_metadata.filenames is None:
yield None, total
continue # dataset case
filenames = json.loads(calc.calc_metadata.filenames.decode('utf-8'))
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment