Commit b1d621f2 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Delete possible repo db entries on deleting an upload.

parent 09f96729
...@@ -82,6 +82,13 @@ class Calc(Base): ...@@ -82,6 +82,13 @@ class Calc(Base):
secondaryjoin=calc_dataset_containment.c.parent_calc_id == coe_calc_id, secondaryjoin=calc_dataset_containment.c.parent_calc_id == coe_calc_id,
backref='children', lazy='subquery', join_depth=1) backref='children', lazy='subquery', join_depth=1)
@staticmethod
def from_calc_id(calc_id: str) -> 'Calc':
repo_db = infrastructure.repository_db
calcs = repo_db.query(Calc).filter_by(checksum=calc_id)
assert calcs.count() <= 1, 'Calc id/checksum must be unique'
return calcs.first()
@classmethod @classmethod
def load_from(cls, obj): def load_from(cls, obj):
repo_db = infrastructure.repository_db repo_db = infrastructure.repository_db
...@@ -247,8 +254,14 @@ class Calc(Base): ...@@ -247,8 +254,14 @@ class Calc(Base):
add_users_to_relation(calc.shared_with, self.shared_with) add_users_to_relation(calc.shared_with, self.shared_with)
# datasets # datasets
calcs_existing_datasets: List[int] = []
for dataset in calc.datasets: for dataset in calc.datasets:
dataset_id = dataset.id dataset_id = dataset.id
if dataset_id in calcs_existing_datasets:
continue
else:
calcs_existing_datasets.append(dataset_id)
coe_dataset_calc: Calc = context.cache(Calc, coe_calc_id=dataset_id) coe_dataset_calc: Calc = context.cache(Calc, coe_calc_id=dataset_id)
if coe_dataset_calc is None: if coe_dataset_calc is None:
coe_dataset_calc = Calc(coe_calc_id=dataset_id) coe_dataset_calc = Calc(coe_calc_id=dataset_id)
......
...@@ -48,6 +48,8 @@ from sqlalchemy import Column, Integer, String, Boolean, DateTime, ForeignKey ...@@ -48,6 +48,8 @@ from sqlalchemy import Column, Integer, String, Boolean, DateTime, ForeignKey
from sqlalchemy.orm import relationship from sqlalchemy.orm import relationship
import filelock import filelock
import os.path import os.path
import warnings
from sqlalchemy import exc as sa_exc
from nomad import utils, infrastructure, config from nomad import utils, infrastructure, config
from nomad.datamodel import UploadWithMetadata from nomad.datamodel import UploadWithMetadata
...@@ -89,7 +91,7 @@ class Upload(Base): # type: ignore ...@@ -89,7 +91,7 @@ class Upload(Base): # type: ignore
created = Column(DateTime) created = Column(DateTime)
user = relationship('User') user = relationship('User')
calcs = relationship('Calc', lazy='subquery') calcs = relationship('Calc', lazy='subquery', passive_deletes=True)
@staticmethod @staticmethod
def from_upload_id(upload_id: str) -> 'Upload': def from_upload_id(upload_id: str) -> 'Upload':
...@@ -110,6 +112,27 @@ class Upload(Base): # type: ignore ...@@ -110,6 +112,27 @@ class Upload(Base): # type: ignore
def upload_time(self) -> Type[datetime.datetime]: def upload_time(self) -> Type[datetime.datetime]:
return self.created return self.created
@staticmethod
def delete(upload_id):
upload = Upload.from_upload_id(upload_id)
if upload is not None:
logger = utils.get_logger(__name__, upload_id=upload.upload_id)
# there is a warning related to SQLalchemy not knowing about the delete cascades
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=sa_exc.SAWarning)
repo_db = infrastructure.repository_db
repo_db.expunge_all()
repo_db.begin()
try:
repo_db.delete(upload)
repo_db.commit()
logger.info('deleted repo upload')
except Exception as e:
logger.error('could not delete repo upload', exc_info=e)
repo_db.rollback()
@staticmethod @staticmethod
def publish(upload: UploadWithMetadata) -> int: def publish(upload: UploadWithMetadata) -> int:
""" """
......
...@@ -1655,7 +1655,7 @@ ALTER TABLE ONLY public.atoms ...@@ -1655,7 +1655,7 @@ ALTER TABLE ONLY public.atoms
-- --
ALTER TABLE ONLY public.basis_sets ALTER TABLE ONLY public.basis_sets
ADD CONSTRAINT basis_sets_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id); ADD CONSTRAINT basis_sets_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
-- --
...@@ -1663,7 +1663,7 @@ ALTER TABLE ONLY public.basis_sets ...@@ -1663,7 +1663,7 @@ ALTER TABLE ONLY public.basis_sets
-- --
ALTER TABLE ONLY public.calcsets ALTER TABLE ONLY public.calcsets
ADD CONSTRAINT calcsets_children_calc_id_fkey FOREIGN KEY (children_calc_id) REFERENCES public.calculations(calc_id); ADD CONSTRAINT calcsets_children_calc_id_fkey FOREIGN KEY (children_calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
-- --
...@@ -1671,7 +1671,7 @@ ALTER TABLE ONLY public.calcsets ...@@ -1671,7 +1671,7 @@ ALTER TABLE ONLY public.calcsets
-- --
ALTER TABLE ONLY public.calcsets ALTER TABLE ONLY public.calcsets
ADD CONSTRAINT calcsets_parent_calc_id_fkey FOREIGN KEY (parent_calc_id) REFERENCES public.calculations(calc_id); ADD CONSTRAINT calcsets_parent_calc_id_fkey FOREIGN KEY (parent_calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
-- --
...@@ -1679,7 +1679,7 @@ ALTER TABLE ONLY public.calcsets ...@@ -1679,7 +1679,7 @@ ALTER TABLE ONLY public.calcsets
-- --
ALTER TABLE ONLY public.calculations ALTER TABLE ONLY public.calculations
ADD CONSTRAINT calculations_origin_id_fkey FOREIGN KEY (origin_id) REFERENCES public.uploads(upload_id); ADD CONSTRAINT calculations_origin_id_fkey FOREIGN KEY (origin_id) REFERENCES public.uploads(upload_id) ON DELETE CASCADE;
-- --
...@@ -1695,7 +1695,7 @@ ALTER TABLE ONLY public.calculations ...@@ -1695,7 +1695,7 @@ ALTER TABLE ONLY public.calculations
-- --
ALTER TABLE ONLY public.charges ALTER TABLE ONLY public.charges
ADD CONSTRAINT charges_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id); ADD CONSTRAINT charges_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
-- --
...@@ -1703,7 +1703,7 @@ ALTER TABLE ONLY public.charges ...@@ -1703,7 +1703,7 @@ ALTER TABLE ONLY public.charges
-- --
ALTER TABLE ONLY public.coauthorships ALTER TABLE ONLY public.coauthorships
ADD CONSTRAINT coauthorships_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id); ADD CONSTRAINT coauthorships_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
-- --
...@@ -1727,7 +1727,7 @@ ALTER TABLE ONLY public.codeversions ...@@ -1727,7 +1727,7 @@ ALTER TABLE ONLY public.codeversions
-- --
ALTER TABLE ONLY public.eigenvalues ALTER TABLE ONLY public.eigenvalues
ADD CONSTRAINT eigenvalues_electrons_calc_id_fkey FOREIGN KEY (electrons_calc_id) REFERENCES public.electrons(calc_id); ADD CONSTRAINT eigenvalues_electrons_calc_id_fkey FOREIGN KEY (electrons_calc_id) REFERENCES public.electrons(calc_id) ON DELETE CASCADE;
-- --
...@@ -1735,7 +1735,7 @@ ALTER TABLE ONLY public.eigenvalues ...@@ -1735,7 +1735,7 @@ ALTER TABLE ONLY public.eigenvalues
-- --
ALTER TABLE ONLY public.eigenvalues ALTER TABLE ONLY public.eigenvalues
ADD CONSTRAINT eigenvalues_phonons_calc_id_fkey FOREIGN KEY (phonons_calc_id) REFERENCES public.phonons(calc_id); ADD CONSTRAINT eigenvalues_phonons_calc_id_fkey FOREIGN KEY (phonons_calc_id) REFERENCES public.phonons(calc_id) ON DELETE CASCADE;
-- --
...@@ -1743,7 +1743,7 @@ ALTER TABLE ONLY public.eigenvalues ...@@ -1743,7 +1743,7 @@ ALTER TABLE ONLY public.eigenvalues
-- --
ALTER TABLE ONLY public.electrons ALTER TABLE ONLY public.electrons
ADD CONSTRAINT electrons_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id); ADD CONSTRAINT electrons_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
-- --
...@@ -1751,7 +1751,7 @@ ALTER TABLE ONLY public.electrons ...@@ -1751,7 +1751,7 @@ ALTER TABLE ONLY public.electrons
-- --
ALTER TABLE ONLY public.energies ALTER TABLE ONLY public.energies
ADD CONSTRAINT energies_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id); ADD CONSTRAINT energies_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
-- --
...@@ -1759,7 +1759,7 @@ ALTER TABLE ONLY public.energies ...@@ -1759,7 +1759,7 @@ ALTER TABLE ONLY public.energies
-- --
ALTER TABLE ONLY public.forces ALTER TABLE ONLY public.forces
ADD CONSTRAINT forces_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id); ADD CONSTRAINT forces_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
-- --
...@@ -1767,7 +1767,7 @@ ALTER TABLE ONLY public.forces ...@@ -1767,7 +1767,7 @@ ALTER TABLE ONLY public.forces
-- --
ALTER TABLE ONLY public.grid ALTER TABLE ONLY public.grid
ADD CONSTRAINT grid_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id); ADD CONSTRAINT grid_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
-- --
...@@ -1791,7 +1791,7 @@ ALTER TABLE ONLY public.login_tokens ...@@ -1791,7 +1791,7 @@ ALTER TABLE ONLY public.login_tokens
-- --
ALTER TABLE ONLY public.metadata ALTER TABLE ONLY public.metadata
ADD CONSTRAINT metadata_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id); ADD CONSTRAINT metadata_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
-- --
...@@ -1799,7 +1799,7 @@ ALTER TABLE ONLY public.metadata ...@@ -1799,7 +1799,7 @@ ALTER TABLE ONLY public.metadata
-- --
ALTER TABLE ONLY public.metadata_citations ALTER TABLE ONLY public.metadata_citations
ADD CONSTRAINT metadata_citations_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id); ADD CONSTRAINT metadata_citations_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
-- --
...@@ -1823,7 +1823,7 @@ ALTER TABLE ONLY public.metadata ...@@ -1823,7 +1823,7 @@ ALTER TABLE ONLY public.metadata
-- --
ALTER TABLE ONLY public.ownerships ALTER TABLE ONLY public.ownerships
ADD CONSTRAINT ownerships_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id); ADD CONSTRAINT ownerships_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
-- --
...@@ -1839,7 +1839,7 @@ ALTER TABLE ONLY public.ownerships ...@@ -1839,7 +1839,7 @@ ALTER TABLE ONLY public.ownerships
-- --
ALTER TABLE ONLY public.phonons ALTER TABLE ONLY public.phonons
ADD CONSTRAINT phonons_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id); ADD CONSTRAINT phonons_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
-- --
...@@ -1847,7 +1847,7 @@ ALTER TABLE ONLY public.phonons ...@@ -1847,7 +1847,7 @@ ALTER TABLE ONLY public.phonons
-- --
ALTER TABLE ONLY public.recipintegs ALTER TABLE ONLY public.recipintegs
ADD CONSTRAINT recipintegs_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id); ADD CONSTRAINT recipintegs_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
-- --
...@@ -1863,7 +1863,7 @@ ALTER TABLE ONLY public.sessions ...@@ -1863,7 +1863,7 @@ ALTER TABLE ONLY public.sessions
-- --
ALTER TABLE ONLY public.shareships ALTER TABLE ONLY public.shareships
ADD CONSTRAINT shareships_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id); ADD CONSTRAINT shareships_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
-- --
...@@ -1879,7 +1879,7 @@ ALTER TABLE ONLY public.shareships ...@@ -1879,7 +1879,7 @@ ALTER TABLE ONLY public.shareships
-- --
ALTER TABLE ONLY public.spacegroups ALTER TABLE ONLY public.spacegroups
ADD CONSTRAINT spacegroups_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id); ADD CONSTRAINT spacegroups_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
-- --
...@@ -1887,7 +1887,7 @@ ALTER TABLE ONLY public.spacegroups ...@@ -1887,7 +1887,7 @@ ALTER TABLE ONLY public.spacegroups
-- --
ALTER TABLE ONLY public.struct_optimisation ALTER TABLE ONLY public.struct_optimisation
ADD CONSTRAINT struct_optimisation_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id); ADD CONSTRAINT struct_optimisation_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
-- --
...@@ -1895,7 +1895,7 @@ ALTER TABLE ONLY public.struct_optimisation ...@@ -1895,7 +1895,7 @@ ALTER TABLE ONLY public.struct_optimisation
-- --
ALTER TABLE ONLY public.struct_ratios ALTER TABLE ONLY public.struct_ratios
ADD CONSTRAINT struct_ratios_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id); ADD CONSTRAINT struct_ratios_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
-- --
...@@ -1903,7 +1903,7 @@ ALTER TABLE ONLY public.struct_ratios ...@@ -1903,7 +1903,7 @@ ALTER TABLE ONLY public.struct_ratios
-- --
ALTER TABLE ONLY public.structures ALTER TABLE ONLY public.structures
ADD CONSTRAINT structures_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id); ADD CONSTRAINT structures_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
-- --
...@@ -1911,7 +1911,7 @@ ALTER TABLE ONLY public.structures ...@@ -1911,7 +1911,7 @@ ALTER TABLE ONLY public.structures
-- --
ALTER TABLE ONLY public.tags ALTER TABLE ONLY public.tags
ADD CONSTRAINT tags_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id); ADD CONSTRAINT tags_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
-- --
...@@ -1935,7 +1935,7 @@ ALTER TABLE ONLY public.uploads ...@@ -1935,7 +1935,7 @@ ALTER TABLE ONLY public.uploads
-- --
ALTER TABLE ONLY public.user_metadata ALTER TABLE ONLY public.user_metadata
ADD CONSTRAINT user_metadata_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id); ADD CONSTRAINT user_metadata_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
-- --
......
...@@ -384,6 +384,12 @@ class Upload(Proc): ...@@ -384,6 +384,12 @@ class Upload(Proc):
logger = self.get_logger() logger = self.get_logger()
with utils.lnr(logger, 'staged upload delete failed'): with utils.lnr(logger, 'staged upload delete failed'):
with utils.timer(
logger, 'upload deleted from repo db', step='repo',
upload_size=self.upload_files.size):
coe_repo.Upload.delete(self.upload_id)
with utils.timer( with utils.timer(
logger, 'upload deleted from index', step='index', logger, 'upload deleted from index', step='index',
upload_size=self.upload_files.size): upload_size=self.upload_files.size):
......
...@@ -14,12 +14,21 @@ ...@@ -14,12 +14,21 @@
import pytest import pytest
from passlib.hash import bcrypt from passlib.hash import bcrypt
from datetime import datetime
from nomad.coe_repo import User, Calc, Upload from nomad.coe_repo import User, Calc, Upload
from nomad.coe_repo.calc import PublishContext from nomad.coe_repo.calc import PublishContext
from nomad import processing, parsing, datamodel from nomad import processing, parsing, datamodel
@pytest.fixture(scope='module')
def example_user_metadata_with_dataset(example_user_metadata) -> dict:
result = dict(**example_user_metadata)
result.update(datasets=[dict(
id=23, _doi='test_doi', _name='test_dataset')])
return result
def assert_user(user, reference): def assert_user(user, reference):
assert user is not None assert user is not None
assert user.user_id == reference.user_id assert user.user_id == reference.user_id
...@@ -94,12 +103,18 @@ def test_add_normalized_calc(postgres, normalized: parsing.LocalBackend, test_us ...@@ -94,12 +103,18 @@ def test_add_normalized_calc(postgres, normalized: parsing.LocalBackend, test_us
def test_add_normalized_calc_with_metadata( def test_add_normalized_calc_with_metadata(
postgres, normalized: parsing.LocalBackend, example_user_metadata: dict): postgres, normalized: parsing.LocalBackend, example_user_metadata_with_dataset: dict):
calc_with_metadata = normalized.to_calc_with_metadata() calc_with_metadata = normalized.to_calc_with_metadata()
calc_with_metadata.files = [calc_with_metadata.mainfile, '1', '2', '3', '4'] calc_with_metadata.files = [calc_with_metadata.mainfile, '1', '2', '3', '4']
calc_with_metadata.apply_user_metadata(example_user_metadata) calc_with_metadata.apply_user_metadata(example_user_metadata_with_dataset)
coe_calc = Calc(coe_calc_id=calc_with_metadata.pid)
coe_upload = Upload(
upload_name='test_upload',
created=datetime.now(),
user_id=0,
is_processed=True)
coe_calc = Calc(coe_calc_id=calc_with_metadata.pid, upload=coe_upload)
coe_calc.apply_calc_with_metadata(calc_with_metadata, PublishContext()) coe_calc.apply_calc_with_metadata(calc_with_metadata, PublishContext())
assert_coe_calc(coe_calc, calc_with_metadata) assert_coe_calc(coe_calc, calc_with_metadata)
...@@ -111,6 +126,23 @@ def test_add_upload(processed: processing.Upload): ...@@ -111,6 +126,23 @@ def test_add_upload(processed: processing.Upload):
assert_coe_upload(processed.upload_id, upload_with_metadata) assert_coe_upload(processed.upload_id, upload_with_metadata)
def test_delete_upload(processed: processing.Upload, example_user_metadata_with_dataset, no_warn):
processed.metadata = example_user_metadata_with_dataset
upload_with_metadata = processed.to_upload_with_metadata()
Upload.publish(upload_with_metadata)
assert_coe_upload(processed.upload_id, upload_with_metadata)
for calc in upload_with_metadata.calcs:
assert Calc.from_calc_id(calc.calc_id) is not None
Upload.delete(processed.upload_id)
assert Upload.from_upload_id(processed.upload_id) is None
for calc in upload_with_metadata.calcs:
assert Calc.from_calc_id(calc.calc_id) is None
Upload.delete(processed.upload_id)
# def test_large_upload(processed: processing.Upload, example_user_metadata): # def test_large_upload(processed: processing.Upload, example_user_metadata):
# processed.metadata = example_user_metadata # processed.metadata = example_user_metadata
# upload_with_metadata = processed.to_upload_with_metadata() # upload_with_metadata = processed.to_upload_with_metadata()
...@@ -136,8 +168,8 @@ def test_add_upload(processed: processing.Upload): ...@@ -136,8 +168,8 @@ def test_add_upload(processed: processing.Upload):
# print('########### %d' % (time.time() - start)) # print('########### %d' % (time.time() - start))
def test_add_upload_with_metadata(processed, example_user_metadata): def test_add_upload_with_metadata(processed, example_user_metadata_with_dataset):
processed.metadata = example_user_metadata processed.metadata = example_user_metadata_with_dataset
upload_with_metadata = processed.to_upload_with_metadata() upload_with_metadata = processed.to_upload_with_metadata()
Upload.publish(upload_with_metadata) Upload.publish(upload_with_metadata)
assert_coe_upload( assert_coe_upload(
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment