Commit 0329dc02 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Use lower repo db isolation level and retry transactions instead.

parent fb4a6ec4
Pipeline #44759 failed with stages
in 22 minutes and 17 seconds
......@@ -42,7 +42,7 @@ This module also provides functionality to add parsed calculation data to the db
:undoc-members:
"""
from typing import Type, Callable
from typing import Type, Callable, Tuple
import datetime
from sqlalchemy import Column, Integer, String, Boolean, DateTime, ForeignKey
from sqlalchemy.orm import relationship
......@@ -124,13 +124,12 @@ class Upload(Base): # type: ignore
"""
assert upload.uploader is not None
logger = utils.get_logger(__name__, upload_id=upload.upload_id)
repo_db = infrastructure.repository_db
repo_db.begin()
logger = utils.get_logger(__name__, upload_id=upload.upload_id)
def fill_publish_transaction() -> Tuple[Upload, bool]:
has_calcs = False
has_calcs = False
try:
# create upload
coe_upload = Upload(
upload_name=upload.upload_id,
......@@ -154,14 +153,44 @@ class Upload(Base): # type: ignore
coe_calc.apply_calc_with_metadata(calc, context=context)
logger.debug('added calculation, not yet committed', calc_id=coe_calc.calc_id)
# commit
def complete(commit: bool) -> int:
return coe_upload, has_calcs
repo_db.begin()
try:
coe_upload, has_calcs = fill_publish_transaction()
except Exception as e:
logger.error('Unexpected exception.', exc_info=e)
repo_db.rollback()
raise e
# commit
def complete(commit: bool) -> int:
upload_to_commit = coe_upload
try:
if commit:
if has_calcs:
# empty upload case
repo_db.commit()
return coe_upload.coe_upload_id
last_error = None
repeat_count = 0
while True:
try:
repo_db.commit()
break
except Exception as e:
repo_db.rollback()
error = str(e)
if last_error != error:
last_error = error
logger.info(
'repeat publish transaction',
error=error, repeat_count=repeat_count)
repo_db.begin()
upload_to_commit, _ = fill_publish_transaction()
repeat_count += 1
else:
raise e
return upload_to_commit.coe_upload_id
else:
# empty upload case
repo_db.rollback()
return -1
......@@ -170,9 +199,9 @@ class Upload(Base): # type: ignore
repo_db.rollback()
logger.info('rolled upload back')
return -1
except Exception as e:
logger.error('Unexpected exception.', exc_info=e)
raise e
return complete
return complete
except Exception as e:
logger.error('Unexpected exception.', exc_info=e)
repo_db.rollback()
raise e
......@@ -182,9 +182,12 @@ def sqlalchemy_repository_db(exists: bool = False, readonly: bool = True, **kwar
params = config.repository_db._asdict()
params.update(**kwargs)
url = 'postgresql://%s:%s@%s:%d/%s' % utils.to_tuple(params, 'user', 'password', 'host', 'port', 'dbname')
# we set a very high isolation level, to prevent conflicts between transactions on the
# We tried to set a very high isolation level, to prevent conflicts between transactions on the
# start-shaped schema, which usually involve read/writes to many tables at once.
engine = create_engine(url, echo=False, isolation_level="SERIALIZABLE")
# Unfortunately, this had week performance, and postgres wasn't even able to serialize on all
# occasions. We are now simply rollingback and retrying on conflicts.
# engine = create_engine(url, echo=False, isolation_level="SERIALIZABLE")
engine = create_engine(url, echo=False)
repository_db_conn = engine.connect()
repository_db = Session(bind=repository_db_conn, autocommit=True)
......
......@@ -111,6 +111,12 @@ def test_add_upload(processed: processing.Upload):
assert_coe_upload(processed.upload_id, upload_with_metadata)
def test_rollback_upload(processed: processing.Upload):
upload_with_metadata = processed.to_upload_with_metadata()
Upload.publish(upload_with_metadata)(False)
assert Upload.from_upload_id(processed.upload_id) is None
# def test_large_upload(processed: processing.Upload, example_user_metadata):
# processed.metadata = example_user_metadata
# upload_with_metadata = processed.to_upload_with_metadata()
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment