Commit be35f5b6 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Bugfix due to mongos bson encode/decod.

parent 7ba22eda
......@@ -176,6 +176,22 @@ class UploadProc(ProcPipeline):
if not self.is_started:
self.continue_with(task_names[0])
def update(self, dct):
# Since the data might be updated from deserialized dicts, list and CalcProc
# instances might by dicts, or mongoengines BaseList, BaseDicts. This overwrite
# replaces it.
# TODO there might be a better solution, or the problem solves itself, when
# moving away from mongo.
if 'calc_procs' in dct:
calc_procs = dct['calc_procs']
for idx, calc_proc_dct in enumerate(calc_procs):
if not isinstance(calc_proc_dct, CalcProc):
calc_procs[idx] = CalcProc(**calc_proc_dct)
if type(calc_procs) != list:
dct['calc_procs'] = list(calc_procs)
super().update(dct)
@property
def _celery_task_result(self) -> AsyncResult:
"""
......
......@@ -65,10 +65,10 @@ def extracting_task(task: Task, proc: UploadProc) -> UploadProc:
for parser in parsers:
if parser.is_mainfile(upload, filename):
tmp_mainfile = upload.get_path(filename)
calc_processing = CalcProc(
calc_proc = CalcProc(
proc.upload_hash, filename, parser.name, tmp_mainfile)
proc.calc_procs.append(calc_proc)
proc.calc_procs.append(calc_processing)
except files.UploadError as e:
logger.warn('Could find parse specs in open upload', exc_info=e)
proc.fail(e)
......@@ -116,10 +116,8 @@ def parse_all_task(task: Task, upload_proc: UploadProc, cleanup: Signature) -> U
parses = group(parse_task.s(calc_proc) for calc_proc in upload_proc.calc_procs)
# save the calc processing task ids to the overall processing
i = 0
for child in parses.freeze().children:
upload_proc.calc_procs[i].celery_task_id = child.task_id
i = i + 1
for idx, child in enumerate(parses.freeze().children):
upload_proc.calc_procs[idx].celery_task_id = child.task_id
# initiate the chord that runs calc processings first, and close_upload afterwards
chord(parses)(cleanup.clone(args=(upload_proc,)))
......
......@@ -13,7 +13,7 @@ from tests.test_processing import example_files
from tests.test_files import assert_exists
# import fixtures
from tests.test_files import clear_files, archive_id # pylint: disable=unused-import
from tests.test_processing import celery_session_worker, celery_config, celery_includes # pylint: disable=unused-import
from tests.test_processing import celery_config, celery_includes # pylint: disable=unused-import
@pytest.fixture
......
......@@ -34,9 +34,9 @@ from tests.test_files import clear_files # pylint: disable=unused-import
example_files = [empty_file, example_file]
@pytest.fixture(scope='session')
def celery_session_worker():
return app
# @pytest.fixture(scope='session')
# def celery_session_worker():
# return app
@pytest.fixture(scope='session')
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment