Added tests for user metadata migration.

nomad/api/upload.py

@@ -57,8 +57,8 @@ metadata_model = api.model('MetaData', {
     'references': fields.List(fields.String, descriptions='References allow to link calculations to external source, e.g. URLs.'),
     'coauthors': fields.List(fields.String, description='A list of co-authors given by user_id.'),
     'shared_with': fields.List(fields.String, description='A list of users to share calculations with given by user_id.'),
-    '_upload_time': fields.List(fields.DateTime(dt_format='iso8601'), description='Overrride the upload time.'),
-    '_uploader': fields.List(fields.String, description='Override the uploader with the given user id.')
+    '_upload_time': fields.DateTime(dt_format='iso8601', description='Overrride the upload time.'),
+    '_uploader': fields.String(description='Override the uploader with the given user id.')
 })
 
 calc_metadata_model = api.inherit('CalcMetaData', metadata_model, {
@@ -346,7 +346,6 @@ class UploadResource(Resource):
                 abort(400, message='Cannot publish an upload that failed processing')
             try:
                 upload.metadata = metadata
-                print('ÜÜÜÜÜÜÜÜ ' + str(upload.metadata))
                 upload.publish_upload()
             except ProcessAlreadyRunning:
                 abort(400, message='The upload is still/already processed')

nomad/coe_repo/calc.py

@@ -178,10 +178,7 @@ class Calc(Base, datamodel.Calc):  # type: ignore
         result.coauthors = list(user.user_id for user in self.coauthors)
         result.shared_with = list(user.user_id for user in self.shared_with)
 
-        return {
-            key: value for key, value in result.items()
-            if value is not None and value != []
-        }
+        return result
 
 
 CalcWithMetadata.register_mapping(Calc, Calc.to_calc_with_metadata)

nomad/migration.py

@@ -31,7 +31,6 @@ from passlib.hash import bcrypt
 from werkzeug.contrib.iterio import IterIO
 import time
 from bravado.exception import HTTPNotFound
-from datetime import datetime
 
 from nomad import utils, config, infrastructure
 from nomad.files import repo_data_to_calc_with_metadata
@@ -305,9 +304,10 @@ class NomadCOEMigration:
             metadata_dict = dict()
             upload_metadata = dict(calculations=upload_metadata_calcs)
             for source_calc in SourceCalc.objects(upload=source_upload_id):
-                source_metadata = CalcWithMetadata(upload_id=upload.upload_id, **source_calc.metadata)
+                source_calc.metadata.upload_id = upload.upload_id
+                source_metadata = CalcWithMetadata(**source_calc.metadata)
                 source_metadata.mainfile = source_calc.mainfile
-                source_metadata.pid = str(source_calc.pid)
+                source_metadata.pid = source_calc.pid
                 source_metadata.__migrated = False
                 upload_metadata_calcs.append(source_metadata)
                 metadata_dict[source_calc.mainfile] = source_metadata
@@ -374,20 +374,26 @@ class NomadCOEMigration:
 
             # publish upload
             admin_keys = ['upload_time', 'uploader', 'pid']
+            user_metadata_keys = [
+                'upload_time', 'uploader', 'pid', 'references', 'datasets', 'mainfile',
+                'with_embargo', 'comment', 'references', 'coauthors', 'shared_with']
 
             def transform(calcWithMetadata):
                 result = dict()
-                print('.::')
                 for key, value in calcWithMetadata.items():
-                    if key in admin_keys:
-                        print('.... ' + key)
-                        target_key = '_%s' % key
-                    else:
-                        target_key = key
+                    if key in user_metadata_keys:
+                        if key in admin_keys:
+                            target_key = '_%s' % key
+                        else:
+                            target_key = key
+
+                        if key in ['pid', 'uploader']:
+                            value = str(value)
+
+                        if key in ['coauthors', 'shared_with']:
+                            value = [str(item) for item in value]
 
-                    if isinstance(value, datetime):
-                        value = value.isoformat()
-                    result[target_key] = value
+                        result[target_key] = value
                 return result
 
             upload_metadata['calculations'] = [

tests/data/migration/example_source_db.sql

@@ -48,15 +48,17 @@ INSERT INTO public.user_metadata VALUES (1, 0, 'label1');
 INSERT INTO public.user_metadata VALUES (2, 1, 'label2');
 INSERT INTO public.ownerships VALUES (1, 1);
 INSERT INTO public.ownerships VALUES (2, 2);
+INSERT INTO public.coauthorships VALUES (1, 2);
+INSERT INTO public.shareships VALUES (2, 1);
 
 -- example dataset
 INSERT INTO public.calculations VALUES (NULL, NULL, NULL, NULL, 1, false, 3, NULL);
 INSERT INTO public.calcsets VALUES (3, 1);
 INSERT INTO public.calcsets VALUES (3, 2);
-INSERT INTO public.citations VALUES(1, 'external_ref', 'EXTERNAL');
+INSERT INTO public.citations VALUES(1, 'internal_ref', 'INTERNAL');
 INSERT INTO public.metadata_citations VALUES (3, 1);
 INSERT INTO public.metadata VALUES (NULL, NULL, NULL, NULL, NULL, 'test_dataset', '2019-01-01 12:00:00', NULL, NULL, 3, NULL);
 
 -- example ref
-INSERT INTO public.citations VALUES(2, 'internal_ref', 'INTERNAL');
+INSERT INTO public.citations VALUES(2, 'external_ref', 'EXTERNAL');
 INSERT INTO public.metadata_citations VALUES (1, 2);
\ No newline at end of file

tests/test_migration.py

@@ -18,7 +18,7 @@ import os.path
 from bravado.client import SwaggerClient
 import json
 
-from nomad import infrastructure, coe_repo
+from nomad import infrastructure, coe_repo, datamodel
 
 from nomad.migration import NomadCOEMigration, SourceCalc
 from nomad.infrastructure import repository_db_connection
@@ -166,6 +166,7 @@ mirgation_test_specs = [
 ]
 
 
+@pytest.mark.filterwarnings("ignore: SAWarning")
 @pytest.mark.parametrize('test, assertions', mirgation_test_specs)
 @pytest.mark.timeout(30)
 def test_migrate(migrate_infra, test, assertions, caplog):
@@ -185,13 +186,30 @@ def test_migrate(migrate_infra, test, assertions, caplog):
     assert report['missing_calcs'] == assertions.get('missing', 0)
 
     # assert if migrated calcs have correct user metadata
+    repo_db = infrastructure.repository_db
     if assertions.get('migrated', 0) > 0:
-        repo_db = infrastructure.repository_db
-        for calc in repo_db.query(coe_repo.Calc):
-            print('#### ' + str(calc.coe_calc_id))
         calc_1 = repo_db.query(coe_repo.Calc).get(1)
         assert calc_1 is not None
-        assert len(calc_1.parents) == 1
+        metadata = calc_1.to(datamodel.CalcWithMetadata)
+        assert metadata.pid <= 2
+        assert metadata.uploader == 1
+        assert metadata.upload_time.isoformat() == '2019-01-01T12:00:00+00:00'
+        assert len(metadata.datasets) == 1
+        assert metadata.datasets[0]['id'] == 3
+        assert metadata.datasets[0]['name'] == 'test_dataset'
+        assert metadata.datasets[0]['dois'][0] == 'internal_ref'
+        assert metadata.comment == 'label1'
+        assert len(metadata.coauthors) == 1
+        assert metadata.coauthors[0] == 2
+        assert len(metadata.references) == 1
+        assert metadata.references[0] == 'external_ref'
+
+    if assertions.get('migrated', 0) > 1:
+        calc_2 = repo_db.query(coe_repo.Calc).get(2)
+        assert calc_1 is not None
+        metadata = calc_2.to(datamodel.CalcWithMetadata)
+        assert len(metadata.shared_with) == 1
+        assert metadata.shared_with[0] == 1
 
     errors = 0
     for record in caplog.get_records(when='call'):