Commit ee9b5ba1 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Removed impossible functionality to rollback upload publish.

parent 6e1d3cb4
Pipeline #44798 failed with stages
in 23 minutes and 52 seconds
......@@ -139,13 +139,11 @@ class Upload(Base): # type: ignore
except filelock.Timeout:
logger.warning('could not acquire publish lock after generous timeout')
def release_lock():
publish_filelock.release()
logger.info('released filelock')
repo_db = infrastructure.repository_db
repo_db.expunge_all()
repo_db.begin()
def fill_publish_transaction() -> Tuple[Upload, bool]:
try:
has_calcs = False
# create upload
......@@ -172,17 +170,26 @@ class Upload(Base): # type: ignore
logger.debug('added calculation, not yet committed', calc_id=coe_calc.calc_id)
logger.info('filled publish transaction')
return coe_upload, has_calcs
repo_db.expunge_all()
repo_db.begin()
try:
coe_upload, has_calcs = fill_publish_transaction()
upload_id = -1
if has_calcs:
repo_db.commit()
logger.info('committed publish transaction')
upload_id = coe_upload.coe_upload_id
else:
# empty upload case
repo_db.rollback()
return -1
logger.info('added upload')
return upload_id
except Exception as e:
logger.error('Unexpected exception.', exc_info=e)
repo_db.rollback()
release_lock()
raise e
finally:
publish_filelock.release()
logger.info('released filelock')
# commit
def complete(commit: bool) -> int:
......
......@@ -400,41 +400,36 @@ class Upload(Proc):
with utils.timer(
logger, 'upload added to repository', step='repo',
upload_size=self.upload_files.size):
upload_transaction_complete = coe_repo.Upload.publish(upload_with_metadata)
coe_repo.Upload.publish(upload_with_metadata)
try:
with utils.timer(
logger, 'staged upload files packed', step='pack',
upload_size=self.upload_files.size):
coe_upload = coe_repo.Upload.from_upload_id(upload_with_metadata.upload_id)
if coe_upload is not None:
for coe_calc in coe_upload.calcs:
calc_metadata = coe_calc.to_calc_with_metadata()
calc_metadata.published = True
self.upload_files.metadata.update(
calc_id=calc_metadata.calc_id, updates=calc_metadata.to_dict())
logger.info('metadata updated after publish to coe repo', step='publish')
self.upload_files.pack()
with utils.timer(
logger, 'index updated', step='index',
upload_size=self.upload_files.size):
coe_upload = coe_repo.Upload.from_upload_id(upload_with_metadata.upload_id)
if coe_upload is not None:
search.publish(
[coe_calc.to_calc_with_metadata() for coe_calc in coe_upload.calcs])
with utils.timer(
logger, 'staged upload deleted', step='delete staged',
upload_size=self.upload_files.size):
self.upload_files.delete()
self.delete()
except Exception as e:
upload_transaction_complete(False)
raise e
with utils.timer(
logger, 'staged upload files packed', step='pack',
upload_size=self.upload_files.size):
coe_upload = coe_repo.Upload.from_upload_id(upload_with_metadata.upload_id)
if coe_upload is not None:
for coe_calc in coe_upload.calcs:
calc_metadata = coe_calc.to_calc_with_metadata()
calc_metadata.published = True
self.upload_files.metadata.update(
calc_id=calc_metadata.calc_id, updates=calc_metadata.to_dict())
logger.info('metadata updated after publish to coe repo', step='publish')
self.upload_files.pack()
with utils.timer(
logger, 'index updated', step='index',
upload_size=self.upload_files.size):
coe_upload = coe_repo.Upload.from_upload_id(upload_with_metadata.upload_id)
if coe_upload is not None:
search.publish(
[coe_calc.to_calc_with_metadata() for coe_calc in coe_upload.calcs])
with utils.timer(
logger, 'staged upload deleted', step='delete staged',
upload_size=self.upload_files.size):
self.upload_files.delete()
self.delete()
upload_transaction_complete(True)
return True # do not save the process status on the delete upload
@process
......
......@@ -107,17 +107,7 @@ def test_add_normalized_calc_with_metadata(
def test_add_upload(processed: processing.Upload):
upload_with_metadata = processed.to_upload_with_metadata()
Upload.publish(upload_with_metadata)(True)
assert_coe_upload(processed.upload_id, upload_with_metadata)
def test_rollback_upload(processed: processing.Upload, postgres):
assert Upload.from_upload_id(processed.upload_id) is None
upload_with_metadata = processed.to_upload_with_metadata()
assert Upload.publish(upload_with_metadata)(False) == -1
assert Upload.from_upload_id(processed.upload_id) is None
Upload.publish(upload_with_metadata)(True)
Upload.publish(upload_with_metadata)
assert_coe_upload(processed.upload_id, upload_with_metadata)
......@@ -142,14 +132,14 @@ def test_rollback_upload(processed: processing.Upload, postgres):
# import time
# start = time.time()
# upload_with_metadata.calcs = many_calcs()
# Upload.publish(upload_with_metadata)(True)
# Upload.publish(upload_with_metadata)
# print('########### %d' % (time.time() - start))
def test_add_upload_with_metadata(processed, example_user_metadata):
processed.metadata = example_user_metadata
upload_with_metadata = processed.to_upload_with_metadata()
Upload.publish(upload_with_metadata)(True)
Upload.publish(upload_with_metadata)
assert_coe_upload(
processed.upload_id, upload_with_metadata)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment