Delete possible repo db entries on deleting an upload.

nomad/coe_repo/calc.py

@@ -82,6 +82,13 @@ class Calc(Base):
         secondaryjoin=calc_dataset_containment.c.parent_calc_id == coe_calc_id,
         backref='children', lazy='subquery', join_depth=1)
 
+    @staticmethod
+    def from_calc_id(calc_id: str) -> 'Calc':
+        repo_db = infrastructure.repository_db
+        calcs = repo_db.query(Calc).filter_by(checksum=calc_id)
+        assert calcs.count() <= 1, 'Calc id/checksum must be unique'
+        return calcs.first()
+
     @classmethod
     def load_from(cls, obj):
         repo_db = infrastructure.repository_db
@@ -247,8 +254,14 @@ class Calc(Base):
         add_users_to_relation(calc.shared_with, self.shared_with)
 
         # datasets
+        calcs_existing_datasets: List[int] = []
         for dataset in calc.datasets:
             dataset_id = dataset.id
+            if dataset_id in calcs_existing_datasets:
+                continue
+            else:
+                calcs_existing_datasets.append(dataset_id)
+
             coe_dataset_calc: Calc = context.cache(Calc, coe_calc_id=dataset_id)
             if coe_dataset_calc is None:
                 coe_dataset_calc = Calc(coe_calc_id=dataset_id)

nomad/coe_repo/upload.py

@@ -48,6 +48,8 @@ from sqlalchemy import Column, Integer, String, Boolean, DateTime, ForeignKey
 from sqlalchemy.orm import relationship
 import filelock
 import os.path
+import warnings
+from sqlalchemy import exc as sa_exc
 
 from nomad import utils, infrastructure, config
 from nomad.datamodel import UploadWithMetadata
@@ -89,7 +91,7 @@ class Upload(Base):  # type: ignore
     created = Column(DateTime)
 
     user = relationship('User')
-    calcs = relationship('Calc', lazy='subquery')
+    calcs = relationship('Calc', lazy='subquery', passive_deletes=True)
 
     @staticmethod
     def from_upload_id(upload_id: str) -> 'Upload':
@@ -110,6 +112,27 @@ class Upload(Base):  # type: ignore
     def upload_time(self) -> Type[datetime.datetime]:
         return self.created
 
+    @staticmethod
+    def delete(upload_id):
+        upload = Upload.from_upload_id(upload_id)
+        if upload is not None:
+            logger = utils.get_logger(__name__, upload_id=upload.upload_id)
+
+            # there is a warning related to SQLalchemy not knowing about the delete cascades
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore", category=sa_exc.SAWarning)
+
+                repo_db = infrastructure.repository_db
+                repo_db.expunge_all()
+                repo_db.begin()
+                try:
+                    repo_db.delete(upload)
+                    repo_db.commit()
+                    logger.info('deleted repo upload')
+                except Exception as e:
+                    logger.error('could not delete repo upload', exc_info=e)
+                    repo_db.rollback()
+
     @staticmethod
     def publish(upload: UploadWithMetadata) -> int:
         """

nomad/empty_repository_db.sql

@@ -1655,7 +1655,7 @@ ALTER TABLE ONLY public.atoms
 --
 
 ALTER TABLE ONLY public.basis_sets
-    ADD CONSTRAINT basis_sets_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
+    ADD CONSTRAINT basis_sets_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
 
 
 --
@@ -1663,7 +1663,7 @@ ALTER TABLE ONLY public.basis_sets
 --
 
 ALTER TABLE ONLY public.calcsets
-    ADD CONSTRAINT calcsets_children_calc_id_fkey FOREIGN KEY (children_calc_id) REFERENCES public.calculations(calc_id);
+    ADD CONSTRAINT calcsets_children_calc_id_fkey FOREIGN KEY (children_calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
 
 
 --
@@ -1671,7 +1671,7 @@ ALTER TABLE ONLY public.calcsets
 --
 
 ALTER TABLE ONLY public.calcsets
-    ADD CONSTRAINT calcsets_parent_calc_id_fkey FOREIGN KEY (parent_calc_id) REFERENCES public.calculations(calc_id);
+    ADD CONSTRAINT calcsets_parent_calc_id_fkey FOREIGN KEY (parent_calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
 
 
 --
@@ -1679,7 +1679,7 @@ ALTER TABLE ONLY public.calcsets
 --
 
 ALTER TABLE ONLY public.calculations
-    ADD CONSTRAINT calculations_origin_id_fkey FOREIGN KEY (origin_id) REFERENCES public.uploads(upload_id);
+    ADD CONSTRAINT calculations_origin_id_fkey FOREIGN KEY (origin_id) REFERENCES public.uploads(upload_id) ON DELETE CASCADE;
 
 
 --
@@ -1695,7 +1695,7 @@ ALTER TABLE ONLY public.calculations
 --
 
 ALTER TABLE ONLY public.charges
-    ADD CONSTRAINT charges_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
+    ADD CONSTRAINT charges_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
 
 
 --
@@ -1703,7 +1703,7 @@ ALTER TABLE ONLY public.charges
 --
 
 ALTER TABLE ONLY public.coauthorships
-    ADD CONSTRAINT coauthorships_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
+    ADD CONSTRAINT coauthorships_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
 
 
 --
@@ -1727,7 +1727,7 @@ ALTER TABLE ONLY public.codeversions
 --
 
 ALTER TABLE ONLY public.eigenvalues
-    ADD CONSTRAINT eigenvalues_electrons_calc_id_fkey FOREIGN KEY (electrons_calc_id) REFERENCES public.electrons(calc_id);
+    ADD CONSTRAINT eigenvalues_electrons_calc_id_fkey FOREIGN KEY (electrons_calc_id) REFERENCES public.electrons(calc_id) ON DELETE CASCADE;
 
 
 --
@@ -1735,7 +1735,7 @@ ALTER TABLE ONLY public.eigenvalues
 --
 
 ALTER TABLE ONLY public.eigenvalues
-    ADD CONSTRAINT eigenvalues_phonons_calc_id_fkey FOREIGN KEY (phonons_calc_id) REFERENCES public.phonons(calc_id);
+    ADD CONSTRAINT eigenvalues_phonons_calc_id_fkey FOREIGN KEY (phonons_calc_id) REFERENCES public.phonons(calc_id) ON DELETE CASCADE;
 
 
 --
@@ -1743,7 +1743,7 @@ ALTER TABLE ONLY public.eigenvalues
 --
 
 ALTER TABLE ONLY public.electrons
-    ADD CONSTRAINT electrons_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
+    ADD CONSTRAINT electrons_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
 
 
 --
@@ -1751,7 +1751,7 @@ ALTER TABLE ONLY public.electrons
 --
 
 ALTER TABLE ONLY public.energies
-    ADD CONSTRAINT energies_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
+    ADD CONSTRAINT energies_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
 
 
 --
@@ -1759,7 +1759,7 @@ ALTER TABLE ONLY public.energies
 --
 
 ALTER TABLE ONLY public.forces
-    ADD CONSTRAINT forces_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
+    ADD CONSTRAINT forces_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
 
 
 --
@@ -1767,7 +1767,7 @@ ALTER TABLE ONLY public.forces
 --
 
 ALTER TABLE ONLY public.grid
-    ADD CONSTRAINT grid_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
+    ADD CONSTRAINT grid_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
 
 
 --
@@ -1791,7 +1791,7 @@ ALTER TABLE ONLY public.login_tokens
 --
 
 ALTER TABLE ONLY public.metadata
-    ADD CONSTRAINT metadata_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
+    ADD CONSTRAINT metadata_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
 
 
 --
@@ -1799,7 +1799,7 @@ ALTER TABLE ONLY public.metadata
 --
 
 ALTER TABLE ONLY public.metadata_citations
-    ADD CONSTRAINT metadata_citations_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
+    ADD CONSTRAINT metadata_citations_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
 
 
 --
@@ -1823,7 +1823,7 @@ ALTER TABLE ONLY public.metadata
 --
 
 ALTER TABLE ONLY public.ownerships
-    ADD CONSTRAINT ownerships_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
+    ADD CONSTRAINT ownerships_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
 
 
 --
@@ -1839,7 +1839,7 @@ ALTER TABLE ONLY public.ownerships
 --
 
 ALTER TABLE ONLY public.phonons
-    ADD CONSTRAINT phonons_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
+    ADD CONSTRAINT phonons_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
 
 
 --
@@ -1847,7 +1847,7 @@ ALTER TABLE ONLY public.phonons
 --
 
 ALTER TABLE ONLY public.recipintegs
-    ADD CONSTRAINT recipintegs_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
+    ADD CONSTRAINT recipintegs_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
 
 
 --
@@ -1863,7 +1863,7 @@ ALTER TABLE ONLY public.sessions
 --
 
 ALTER TABLE ONLY public.shareships
-    ADD CONSTRAINT shareships_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
+    ADD CONSTRAINT shareships_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
 
 
 --
@@ -1879,7 +1879,7 @@ ALTER TABLE ONLY public.shareships
 --
 
 ALTER TABLE ONLY public.spacegroups
-    ADD CONSTRAINT spacegroups_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
+    ADD CONSTRAINT spacegroups_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
 
 
 --
@@ -1887,7 +1887,7 @@ ALTER TABLE ONLY public.spacegroups
 --
 
 ALTER TABLE ONLY public.struct_optimisation
-    ADD CONSTRAINT struct_optimisation_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
+    ADD CONSTRAINT struct_optimisation_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
 
 
 --
@@ -1895,7 +1895,7 @@ ALTER TABLE ONLY public.struct_optimisation
 --
 
 ALTER TABLE ONLY public.struct_ratios
-    ADD CONSTRAINT struct_ratios_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
+    ADD CONSTRAINT struct_ratios_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
 
 
 --
@@ -1903,7 +1903,7 @@ ALTER TABLE ONLY public.struct_ratios
 --
 
 ALTER TABLE ONLY public.structures
-    ADD CONSTRAINT structures_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
+    ADD CONSTRAINT structures_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
 
 
 --
@@ -1911,7 +1911,7 @@ ALTER TABLE ONLY public.structures
 --
 
 ALTER TABLE ONLY public.tags
-    ADD CONSTRAINT tags_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
+    ADD CONSTRAINT tags_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
 
 
 --
@@ -1935,7 +1935,7 @@ ALTER TABLE ONLY public.uploads
 --
 
 ALTER TABLE ONLY public.user_metadata
-    ADD CONSTRAINT user_metadata_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
+    ADD CONSTRAINT user_metadata_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
 
 
 --

nomad/processing/data.py

@@ -384,6 +384,12 @@ class Upload(Proc):
         logger = self.get_logger()
 
         with utils.lnr(logger, 'staged upload delete failed'):
+
+            with utils.timer(
+                    logger, 'upload deleted from repo db', step='repo',
+                    upload_size=self.upload_files.size):
+                coe_repo.Upload.delete(self.upload_id)
+
             with utils.timer(
                     logger, 'upload deleted from index', step='index',
                     upload_size=self.upload_files.size):

tests/test_coe_repo.py

@@ -14,12 +14,21 @@
 
 import pytest
 from passlib.hash import bcrypt
+from datetime import datetime
 
 from nomad.coe_repo import User, Calc, Upload
 from nomad.coe_repo.calc import PublishContext
 from nomad import processing, parsing, datamodel
 
 
+@pytest.fixture(scope='module')
+def example_user_metadata_with_dataset(example_user_metadata) -> dict:
+    result = dict(**example_user_metadata)
+    result.update(datasets=[dict(
+        id=23, _doi='test_doi', _name='test_dataset')])
+    return result
+
+
 def assert_user(user, reference):
     assert user is not None
     assert user.user_id == reference.user_id
@@ -94,12 +103,18 @@ def test_add_normalized_calc(postgres, normalized: parsing.LocalBackend, test_us
 
 
 def test_add_normalized_calc_with_metadata(
-        postgres, normalized: parsing.LocalBackend, example_user_metadata: dict):
+        postgres, normalized: parsing.LocalBackend, example_user_metadata_with_dataset: dict):
 
     calc_with_metadata = normalized.to_calc_with_metadata()
     calc_with_metadata.files = [calc_with_metadata.mainfile, '1', '2', '3', '4']
-    calc_with_metadata.apply_user_metadata(example_user_metadata)
-    coe_calc = Calc(coe_calc_id=calc_with_metadata.pid)
+    calc_with_metadata.apply_user_metadata(example_user_metadata_with_dataset)
+
+    coe_upload = Upload(
+        upload_name='test_upload',
+        created=datetime.now(),
+        user_id=0,
+        is_processed=True)
+    coe_calc = Calc(coe_calc_id=calc_with_metadata.pid, upload=coe_upload)
     coe_calc.apply_calc_with_metadata(calc_with_metadata, PublishContext())
 
     assert_coe_calc(coe_calc, calc_with_metadata)
@@ -111,6 +126,23 @@ def test_add_upload(processed: processing.Upload):
     assert_coe_upload(processed.upload_id, upload_with_metadata)
 
 
+def test_delete_upload(processed: processing.Upload, example_user_metadata_with_dataset, no_warn):
+    processed.metadata = example_user_metadata_with_dataset
+    upload_with_metadata = processed.to_upload_with_metadata()
+    Upload.publish(upload_with_metadata)
+    assert_coe_upload(processed.upload_id, upload_with_metadata)
+
+    for calc in upload_with_metadata.calcs:
+        assert Calc.from_calc_id(calc.calc_id) is not None
+
+    Upload.delete(processed.upload_id)
+    assert Upload.from_upload_id(processed.upload_id) is None
+    for calc in upload_with_metadata.calcs:
+        assert Calc.from_calc_id(calc.calc_id) is None
+
+    Upload.delete(processed.upload_id)
+
+
 # def test_large_upload(processed: processing.Upload, example_user_metadata):
 #     processed.metadata = example_user_metadata
 #     upload_with_metadata = processed.to_upload_with_metadata()
@@ -136,8 +168,8 @@ def test_add_upload(processed: processing.Upload):
 #     print('########### %d' % (time.time() - start))
 
 
-def test_add_upload_with_metadata(processed, example_user_metadata):
-    processed.metadata = example_user_metadata
+def test_add_upload_with_metadata(processed, example_user_metadata_with_dataset):
+    processed.metadata = example_user_metadata_with_dataset
     upload_with_metadata = processed.to_upload_with_metadata()
     Upload.publish(upload_with_metadata)
     assert_coe_upload(