Commit e6de79a1 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch 'migration' of gitlab.mpcdf.mpg.de:nomad-lab/nomad-FAIR into migration

parents 719d77e5 10b695c9
Pipeline #45369 passed with stages
in 30 minutes and 46 seconds
......@@ -82,6 +82,13 @@ class Calc(Base):
secondaryjoin=calc_dataset_containment.c.parent_calc_id == coe_calc_id,
backref='children', lazy='subquery', join_depth=1)
@staticmethod
def from_calc_id(calc_id: str) -> 'Calc':
repo_db = infrastructure.repository_db
calcs = repo_db.query(Calc).filter_by(checksum=calc_id)
assert calcs.count() <= 1, 'Calc id/checksum must be unique'
return calcs.first()
@classmethod
def load_from(cls, obj):
repo_db = infrastructure.repository_db
......@@ -247,8 +254,14 @@ class Calc(Base):
add_users_to_relation(calc.shared_with, self.shared_with)
# datasets
calcs_existing_datasets: List[int] = []
for dataset in calc.datasets:
dataset_id = dataset.id
if dataset_id in calcs_existing_datasets:
continue
else:
calcs_existing_datasets.append(dataset_id)
coe_dataset_calc: Calc = context.cache(Calc, coe_calc_id=dataset_id)
if coe_dataset_calc is None:
coe_dataset_calc = Calc(coe_calc_id=dataset_id)
......
......@@ -48,6 +48,8 @@ from sqlalchemy import Column, Integer, String, Boolean, DateTime, ForeignKey
from sqlalchemy.orm import relationship
import filelock
import os.path
import warnings
from sqlalchemy import exc as sa_exc
from nomad import utils, infrastructure, config
from nomad.datamodel import UploadWithMetadata
......@@ -89,7 +91,7 @@ class Upload(Base): # type: ignore
created = Column(DateTime)
user = relationship('User')
calcs = relationship('Calc', lazy='subquery')
calcs = relationship('Calc', lazy='subquery', passive_deletes=True)
@staticmethod
def from_upload_id(upload_id: str) -> 'Upload':
......@@ -110,6 +112,27 @@ class Upload(Base): # type: ignore
def upload_time(self) -> Type[datetime.datetime]:
return self.created
@staticmethod
def delete(upload_id):
upload = Upload.from_upload_id(upload_id)
if upload is not None:
logger = utils.get_logger(__name__, upload_id=upload.upload_id)
# there is a warning related to SQLalchemy not knowing about the delete cascades
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=sa_exc.SAWarning)
repo_db = infrastructure.repository_db
repo_db.expunge_all()
repo_db.begin()
try:
repo_db.delete(upload)
repo_db.commit()
logger.info('deleted repo upload')
except Exception as e:
logger.error('could not delete repo upload', exc_info=e)
repo_db.rollback()
@staticmethod
def publish(upload: UploadWithMetadata) -> int:
"""
......
......@@ -1655,7 +1655,7 @@ ALTER TABLE ONLY public.atoms
--
ALTER TABLE ONLY public.basis_sets
ADD CONSTRAINT basis_sets_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
ADD CONSTRAINT basis_sets_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
--
......@@ -1663,7 +1663,7 @@ ALTER TABLE ONLY public.basis_sets
--
ALTER TABLE ONLY public.calcsets
ADD CONSTRAINT calcsets_children_calc_id_fkey FOREIGN KEY (children_calc_id) REFERENCES public.calculations(calc_id);
ADD CONSTRAINT calcsets_children_calc_id_fkey FOREIGN KEY (children_calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
--
......@@ -1671,7 +1671,7 @@ ALTER TABLE ONLY public.calcsets
--
ALTER TABLE ONLY public.calcsets
ADD CONSTRAINT calcsets_parent_calc_id_fkey FOREIGN KEY (parent_calc_id) REFERENCES public.calculations(calc_id);
ADD CONSTRAINT calcsets_parent_calc_id_fkey FOREIGN KEY (parent_calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
--
......@@ -1679,7 +1679,7 @@ ALTER TABLE ONLY public.calcsets
--
ALTER TABLE ONLY public.calculations
ADD CONSTRAINT calculations_origin_id_fkey FOREIGN KEY (origin_id) REFERENCES public.uploads(upload_id);
ADD CONSTRAINT calculations_origin_id_fkey FOREIGN KEY (origin_id) REFERENCES public.uploads(upload_id) ON DELETE CASCADE;
--
......@@ -1695,7 +1695,7 @@ ALTER TABLE ONLY public.calculations
--
ALTER TABLE ONLY public.charges
ADD CONSTRAINT charges_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
ADD CONSTRAINT charges_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
--
......@@ -1703,7 +1703,7 @@ ALTER TABLE ONLY public.charges
--
ALTER TABLE ONLY public.coauthorships
ADD CONSTRAINT coauthorships_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
ADD CONSTRAINT coauthorships_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
--
......@@ -1727,7 +1727,7 @@ ALTER TABLE ONLY public.codeversions
--
ALTER TABLE ONLY public.eigenvalues
ADD CONSTRAINT eigenvalues_electrons_calc_id_fkey FOREIGN KEY (electrons_calc_id) REFERENCES public.electrons(calc_id);
ADD CONSTRAINT eigenvalues_electrons_calc_id_fkey FOREIGN KEY (electrons_calc_id) REFERENCES public.electrons(calc_id) ON DELETE CASCADE;
--
......@@ -1735,7 +1735,7 @@ ALTER TABLE ONLY public.eigenvalues
--
ALTER TABLE ONLY public.eigenvalues
ADD CONSTRAINT eigenvalues_phonons_calc_id_fkey FOREIGN KEY (phonons_calc_id) REFERENCES public.phonons(calc_id);
ADD CONSTRAINT eigenvalues_phonons_calc_id_fkey FOREIGN KEY (phonons_calc_id) REFERENCES public.phonons(calc_id) ON DELETE CASCADE;
--
......@@ -1743,7 +1743,7 @@ ALTER TABLE ONLY public.eigenvalues
--
ALTER TABLE ONLY public.electrons
ADD CONSTRAINT electrons_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
ADD CONSTRAINT electrons_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
--
......@@ -1751,7 +1751,7 @@ ALTER TABLE ONLY public.electrons
--
ALTER TABLE ONLY public.energies
ADD CONSTRAINT energies_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
ADD CONSTRAINT energies_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
--
......@@ -1759,7 +1759,7 @@ ALTER TABLE ONLY public.energies
--
ALTER TABLE ONLY public.forces
ADD CONSTRAINT forces_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
ADD CONSTRAINT forces_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
--
......@@ -1767,7 +1767,7 @@ ALTER TABLE ONLY public.forces
--
ALTER TABLE ONLY public.grid
ADD CONSTRAINT grid_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
ADD CONSTRAINT grid_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
--
......@@ -1791,7 +1791,7 @@ ALTER TABLE ONLY public.login_tokens
--
ALTER TABLE ONLY public.metadata
ADD CONSTRAINT metadata_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
ADD CONSTRAINT metadata_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
--
......@@ -1799,7 +1799,7 @@ ALTER TABLE ONLY public.metadata
--
ALTER TABLE ONLY public.metadata_citations
ADD CONSTRAINT metadata_citations_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
ADD CONSTRAINT metadata_citations_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
--
......@@ -1823,7 +1823,7 @@ ALTER TABLE ONLY public.metadata
--
ALTER TABLE ONLY public.ownerships
ADD CONSTRAINT ownerships_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
ADD CONSTRAINT ownerships_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
--
......@@ -1839,7 +1839,7 @@ ALTER TABLE ONLY public.ownerships
--
ALTER TABLE ONLY public.phonons
ADD CONSTRAINT phonons_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
ADD CONSTRAINT phonons_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
--
......@@ -1847,7 +1847,7 @@ ALTER TABLE ONLY public.phonons
--
ALTER TABLE ONLY public.recipintegs
ADD CONSTRAINT recipintegs_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
ADD CONSTRAINT recipintegs_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
--
......@@ -1863,7 +1863,7 @@ ALTER TABLE ONLY public.sessions
--
ALTER TABLE ONLY public.shareships
ADD CONSTRAINT shareships_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
ADD CONSTRAINT shareships_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
--
......@@ -1879,7 +1879,7 @@ ALTER TABLE ONLY public.shareships
--
ALTER TABLE ONLY public.spacegroups
ADD CONSTRAINT spacegroups_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
ADD CONSTRAINT spacegroups_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
--
......@@ -1887,7 +1887,7 @@ ALTER TABLE ONLY public.spacegroups
--
ALTER TABLE ONLY public.struct_optimisation
ADD CONSTRAINT struct_optimisation_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
ADD CONSTRAINT struct_optimisation_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
--
......@@ -1895,7 +1895,7 @@ ALTER TABLE ONLY public.struct_optimisation
--
ALTER TABLE ONLY public.struct_ratios
ADD CONSTRAINT struct_ratios_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
ADD CONSTRAINT struct_ratios_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
--
......@@ -1903,7 +1903,7 @@ ALTER TABLE ONLY public.struct_ratios
--
ALTER TABLE ONLY public.structures
ADD CONSTRAINT structures_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
ADD CONSTRAINT structures_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
--
......@@ -1911,7 +1911,7 @@ ALTER TABLE ONLY public.structures
--
ALTER TABLE ONLY public.tags
ADD CONSTRAINT tags_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
ADD CONSTRAINT tags_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
--
......@@ -1935,7 +1935,7 @@ ALTER TABLE ONLY public.uploads
--
ALTER TABLE ONLY public.user_metadata
ADD CONSTRAINT user_metadata_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id);
ADD CONSTRAINT user_metadata_calc_id_fkey FOREIGN KEY (calc_id) REFERENCES public.calculations(calc_id) ON DELETE CASCADE;
--
......
......@@ -384,6 +384,12 @@ class Upload(Proc):
logger = self.get_logger()
with utils.lnr(logger, 'staged upload delete failed'):
with utils.timer(
logger, 'upload deleted from repo db', step='repo',
upload_size=self.upload_files.size):
coe_repo.Upload.delete(self.upload_id)
with utils.timer(
logger, 'upload deleted from index', step='index',
upload_size=self.upload_files.size):
......
......@@ -14,12 +14,21 @@
import pytest
from passlib.hash import bcrypt
from datetime import datetime
from nomad.coe_repo import User, Calc, Upload
from nomad.coe_repo.calc import PublishContext
from nomad import processing, parsing, datamodel
@pytest.fixture(scope='module')
def example_user_metadata_with_dataset(example_user_metadata) -> dict:
result = dict(**example_user_metadata)
result.update(datasets=[dict(
id=23, _doi='test_doi', _name='test_dataset')])
return result
def assert_user(user, reference):
assert user is not None
assert user.user_id == reference.user_id
......@@ -94,12 +103,18 @@ def test_add_normalized_calc(postgres, normalized: parsing.LocalBackend, test_us
def test_add_normalized_calc_with_metadata(
postgres, normalized: parsing.LocalBackend, example_user_metadata: dict):
postgres, normalized: parsing.LocalBackend, example_user_metadata_with_dataset: dict):
calc_with_metadata = normalized.to_calc_with_metadata()
calc_with_metadata.files = [calc_with_metadata.mainfile, '1', '2', '3', '4']
calc_with_metadata.apply_user_metadata(example_user_metadata)
coe_calc = Calc(coe_calc_id=calc_with_metadata.pid)
calc_with_metadata.apply_user_metadata(example_user_metadata_with_dataset)
coe_upload = Upload(
upload_name='test_upload',
created=datetime.now(),
user_id=0,
is_processed=True)
coe_calc = Calc(coe_calc_id=calc_with_metadata.pid, upload=coe_upload)
coe_calc.apply_calc_with_metadata(calc_with_metadata, PublishContext())
assert_coe_calc(coe_calc, calc_with_metadata)
......@@ -111,6 +126,23 @@ def test_add_upload(processed: processing.Upload):
assert_coe_upload(processed.upload_id, upload_with_metadata)
def test_delete_upload(processed: processing.Upload, example_user_metadata_with_dataset, no_warn):
processed.metadata = example_user_metadata_with_dataset
upload_with_metadata = processed.to_upload_with_metadata()
Upload.publish(upload_with_metadata)
assert_coe_upload(processed.upload_id, upload_with_metadata)
for calc in upload_with_metadata.calcs:
assert Calc.from_calc_id(calc.calc_id) is not None
Upload.delete(processed.upload_id)
assert Upload.from_upload_id(processed.upload_id) is None
for calc in upload_with_metadata.calcs:
assert Calc.from_calc_id(calc.calc_id) is None
Upload.delete(processed.upload_id)
# def test_large_upload(processed: processing.Upload, example_user_metadata):
# processed.metadata = example_user_metadata
# upload_with_metadata = processed.to_upload_with_metadata()
......@@ -136,8 +168,8 @@ def test_add_upload(processed: processing.Upload):
# print('########### %d' % (time.time() - start))
def test_add_upload_with_metadata(processed, example_user_metadata):
processed.metadata = example_user_metadata
def test_add_upload_with_metadata(processed, example_user_metadata_with_dataset):
processed.metadata = example_user_metadata_with_dataset
upload_with_metadata = processed.to_upload_with_metadata()
Upload.publish(upload_with_metadata)
assert_coe_upload(
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment