Fixed missing propagation of with_embargo during migration.

nomad/processing/data.py

@@ -608,8 +608,8 @@ class Upload(Proc):
             calc_data = upload_files.metadata.get(calc.calc_id)
             calc_with_metadata = CalcWithMetadata(**calc_data)
 
-            calc_metadata = dict(calc_metadatas.get(calc.mainfile, {}))
-            calc_metadata.update(upload_metadata)
+            calc_metadata = dict(upload_metadata)
+            calc_metadata.update(calc_metadatas.get(calc.mainfile, {}))
             calc_with_metadata.apply_user_metadata(calc_metadata)
 
             logger.debug('prepared calc with metadata', calc_id=calc_with_metadata.calc_id)

nomad/search.py

@@ -245,6 +245,7 @@ search_quantities = {
     'quantities': ('term', 'quantities', 'Search for the existence of a certain meta-info quantity'),
     'upload_id': ('term', 'upload_id', 'Search for the upload_id.'),
     'calc_id': ('term', 'calc_id', 'Search for the calc_id.'),
+    'pid': ('term', 'pid', 'Search for the pid.'),
     'mainfile': ('term', 'mainfile', 'Search for the mainfile.')
 }
 """

tests/test_migration.py

@@ -19,8 +19,9 @@ from bravado.client import SwaggerClient
 import json
 import glob
 from io import StringIO
+import bravado.exception
 
-from nomad import infrastructure, coe_repo
+from nomad import infrastructure, coe_repo, utils
 
 from nomad.migration import NomadCOEMigration, SourceCalc, Package
 from nomad.infrastructure import repository_db_connection
@@ -28,6 +29,7 @@ from nomad.infrastructure import repository_db_connection
 from tests.conftest import create_postgres_infra, create_auth_headers
 from tests.bravado_flask import FlaskTestHttpClient
 from tests.test_api import create_auth_headers
+import tests.utils as test_utils
 
 test_source_db_name = 'test_nomad_fairdi_migration_source'
 test_target_db_name = 'test_nomad_fairdi_migration_target'
@@ -155,22 +157,37 @@ def migrate_infra(migration, target_repo, proc_infra, client, monkeypatch):
     assert len(indexed) == 2
 
     # target repo is the infrastructure repo
-    def create_client():
-        admin = target_repo.query(coe_repo.User).filter_by(email='admin').first()
-        http_client = FlaskTestHttpClient(client, headers=create_auth_headers(admin))
+    monkeypatch.setattr('nomad.infrastructure.repository_db', target_repo)
+
+    infra = utils.POPO()
+
+    infra.admin_auth = create_auth_headers(coe_repo.User.from_user_id(0))
+
+    def create_client_for_user(auth):
+        http_client = FlaskTestHttpClient(client, headers=auth)
         return SwaggerClient.from_url('/swagger.json', http_client=http_client)
 
+    def create_client():
+        return create_client_for_user(infra.admin_auth)
+
+    monkeypatch.setattr('nomad.client.create_client', create_client)
+
     def stream_upload_with_client(client, upload_f, name=None):
         return client.uploads.upload(file=upload_f, name=name).response().result
 
-    monkeypatch.setattr('nomad.infrastructure.repository_db', target_repo)
-    monkeypatch.setattr('nomad.client.create_client', create_client)
     monkeypatch.setattr('nomad.client.stream_upload_with_client', stream_upload_with_client)
 
     # source repo is the still the original infrastructure repo
     migration.copy_users()
+    infra.one_auth = create_auth_headers(coe_repo.User.from_user_id(3))
+    infra.two_auth = create_auth_headers(coe_repo.User.from_user_id(4))
 
-    yield migration
+    infra.migration = migration
+    infra.flask = client
+    infra.one_client = create_client_for_user(infra.one_auth)
+    infra.two_client = create_client_for_user(infra.one_auth)
+
+    yield infra
 
 
 def test_copy_users(migrate_infra, target_repo):
@@ -213,8 +230,8 @@ def perform_migration_test(migrate_infra, name, test_directory, assertions, kwar
     upload_path = os.path.join(upload_path, os.listdir(upload_path)[0])
 
     pid_prefix = 10
-    migrate_infra.set_pid_prefix(pid_prefix)
-    report = migrate_infra.migrate(upload_path, create_packages=True, **kwargs)
+    migrate_infra.migration.set_pid_prefix(pid_prefix)
+    report = migrate_infra.migration.migrate(upload_path, create_packages=True, **kwargs)
 
     assert report.total_calcs == assertions.get('migrated', 0) + assertions.get('new', 0) + assertions.get('failed', 0)
 
@@ -264,6 +281,31 @@ def perform_migration_test(migrate_infra, name, test_directory, assertions, kwar
 
     assert errors == assertions.get('errors', 0)
 
+    if name == 'baseline':
+        result = migrate_infra.two_client.repo.search(per_page=2, order=1, order_by='pid').response().result
+        assert result.pagination.total == 2
+        calc_1 = result.results[0]
+        assert calc_1['pid'] == '1'
+        assert calc_1['with_embargo'] is False
+
+        calc_2 = result.results[1]
+        assert calc_2['pid'] == '2'
+        assert calc_2['with_embargo'] is True
+
+        # assert if with_embargo is passed through to files
+        with test_utils.assert_exception(bravado.exception.HTTPUnauthorized):
+            migrate_infra.one_client.archive.get_archive_calc(
+                upload_id=calc_2['upload_id'], calc_id=calc_2['calc_id']).response().result
+
+        with test_utils.assert_exception(bravado.exception.HTTPUnauthorized):
+            migrate_infra.one_client.raw.get(
+                upload_id=calc_2['upload_id'], path=calc_2['mainfile']).response().result
+
+        migrate_infra.two_client.archive.get_archive_calc(
+            upload_id=calc_1['upload_id'], calc_id=calc_1['calc_id']).response().result
+        migrate_infra.two_client.raw.get(
+            upload_id=calc_1['upload_id'], path=calc_1['mainfile']).response().result
+
 
 def test_skip_on_same_version(migrate_infra, monkeypatch, caplog):
     assertions = dict(migrated=2, source=2, skipped_packages=0)

tests/utils.py

@@ -14,7 +14,9 @@
 
 """ Methods to help with testing of nomad@FAIRDI."""
 
+from typing import Type
 import json
+from contextlib import contextmanager
 
 
 def assert_log(caplog, level, event_part):
@@ -41,3 +43,18 @@ def assert_log(caplog, level, event_part):
                 # No need to look for more matches since we aren't counting matches.
                 break
     assert(record_receieved)
+
+
+@contextmanager
+def assert_exception(exception_cls: Type = Exception):
+    """
+    A context manager that can be used to assert that the given exception is thrown
+    within the respective ``with``clause.
+    """
+    has_exception = False
+    try:
+        yield
+    except exception_cls:
+        has_exception = True
+
+    assert has_exception