Fixes to migration in cli mirror command.

nomad/cli/client/mirror.py

@@ -37,7 +37,7 @@ def v0Dot6(upload_data):
     """ Inplace transforms v0.6.x upload data into v0.7.x upload data. """
 
     def tarnsform_user_id(source_user_id):
-        target_user = User.repo_users.get(source_user_id)
+        target_user = User.repo_users().get(str(source_user_id))
         if target_user is None:
             __logger.error('user does not exist in target', source_user_id=source_user_id)
             raise KeyError
@@ -61,11 +61,14 @@ def v0Dot6(upload_data):
 
         return target_dataset.dataset_id
 
+    def transform_reference(reference):
+        return reference['value']
+
     upload = json.loads(upload_data.upload)
     upload['user_id'] = tarnsform_user_id(upload['user_id'])
     upload_data.upload = json.dumps(upload)
 
-    for calc_data_json, i in enumerate(upload_data.calcs):
+    for i, calc_data_json in enumerate(upload_data.calcs):
         calc_data = json.loads(calc_data_json)
         metadata = calc_data['metadata']
 
@@ -77,6 +80,9 @@ def v0Dot6(upload_data):
         # transform datasets
         metadata['datasets'] = [transform_dataset(dataset) for dataset in metadata['datasets']]
 
+        # transform references
+        metadata['references'] = [transform_reference(reference) for reference in metadata['references']]
+
         upload_data.calcs[i] = json.dumps(calc_data)
     return upload_data
 

nomad/datamodel/metainfo.py

@@ -70,7 +70,7 @@ class User(metainfo.MSection):
     def repo_users() -> Dict[str, 'User']:
         from nomad import infrastructure
         return {
-            user.repo_user_id: user
+            str(user.repo_user_id): user
             for user in infrastructure.keycloak.search_user()
             if user.repo_user_id is not None
         }

nomad/infrastructure.py

@@ -270,17 +270,18 @@ class Keycloak():
         kwargs = {key: value[0] for key, value in keycloak_user.get('attributes', {}).items()}
         return datamodel.User(
             user_id=keycloak_user['id'],
-            email=keycloak_user['email'],
-            username=keycloak_user.get('username', None),
-            first_name=keycloak_user.get('firstName', None),
-            last_name=keycloak_user.get('lastName', None),
+            email=keycloak_user.get('email'),
+            username=keycloak_user.get('username'),
+            first_name=keycloak_user.get('firstName'),
+            last_name=keycloak_user.get('lastName'),
             created=datetime.fromtimestamp(keycloak_user['createdTimestamp'] / 1000),
             **kwargs)
 
-    def search_user(self, query: str = None):
-        kwargs = {}
+    def search_user(self, query: str = None, **kwargs):
         if query is not None:
             kwargs['query'] = dict(search=query)
+        else:
+            kwargs['query'] = dict(max=1000)
         try:
             keycloak_results = self._admin_client.get_users(**kwargs)
         except Exception as e:

nomad/metainfo/metainfo.py

@@ -805,7 +805,7 @@ class MSection(metaclass=MObjectMeta):
             return bool(value)
 
         else:
-            if type(value) != quantity_def.type:
+            if value is not None and type(value) != quantity_def.type:
                 raise TypeError(
                     'The value %s with type %s for quantity %s is not of type %s' %
                     (value, type(value), quantity_def, quantity_def.type))

ops/deployments/nomad.testing-major.env.sh

+export NOMAD_CLIENT_URL=https://labdev-nomad.esc.rzg.mpg.de/fairdi/nomad/testing-major/api
+export NOMAD_CLIENT_USER=admin
+export NOMAD_FS_LOCAL_TMP=/nomad/fairdi/testing_major/fs/tmp
+export NOMAD_MONGO_DB_NAME=fairdi_nomad_testing_major
+export NOMAD_ELASTIC_INDEX_NAME=fairdi_nomad_testing_major
+export NOMAD_FS_LOCAL_TMP=/nomad/fairdi/testing_major/fs/tmp
+export NOMAD_FS_STAGING=/nomad/fairdi/testing_major/fs/staging
+export NOMAD_FS_PUBLIC=/nomad/fairdi/testing_major/fs/public
+export NOMAD_FS_TMP=/nomad/fairdi/testing_major/fs/tmp
+export NOMAD_SERVICE_ADMIN_USER_ID="82efac55-6187-408c-8027-b98580c0e1c5"
+export NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod
+export NOMAD_FS_PREFIX_SIZE=1