Commit b27e6a7b authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Fixes to migration in cli mirror command.

parent d350168f
Pipeline #63601 passed with stages
in 14 minutes and 16 seconds
...@@ -37,7 +37,7 @@ def v0Dot6(upload_data): ...@@ -37,7 +37,7 @@ def v0Dot6(upload_data):
""" Inplace transforms v0.6.x upload data into v0.7.x upload data. """ """ Inplace transforms v0.6.x upload data into v0.7.x upload data. """
def tarnsform_user_id(source_user_id): def tarnsform_user_id(source_user_id):
target_user = User.repo_users.get(source_user_id) target_user = User.repo_users().get(str(source_user_id))
if target_user is None: if target_user is None:
__logger.error('user does not exist in target', source_user_id=source_user_id) __logger.error('user does not exist in target', source_user_id=source_user_id)
raise KeyError raise KeyError
...@@ -61,11 +61,14 @@ def v0Dot6(upload_data): ...@@ -61,11 +61,14 @@ def v0Dot6(upload_data):
return target_dataset.dataset_id return target_dataset.dataset_id
def transform_reference(reference):
return reference['value']
upload = json.loads(upload_data.upload) upload = json.loads(upload_data.upload)
upload['user_id'] = tarnsform_user_id(upload['user_id']) upload['user_id'] = tarnsform_user_id(upload['user_id'])
upload_data.upload = json.dumps(upload) upload_data.upload = json.dumps(upload)
for calc_data_json, i in enumerate(upload_data.calcs): for i, calc_data_json in enumerate(upload_data.calcs):
calc_data = json.loads(calc_data_json) calc_data = json.loads(calc_data_json)
metadata = calc_data['metadata'] metadata = calc_data['metadata']
...@@ -77,6 +80,9 @@ def v0Dot6(upload_data): ...@@ -77,6 +80,9 @@ def v0Dot6(upload_data):
# transform datasets # transform datasets
metadata['datasets'] = [transform_dataset(dataset) for dataset in metadata['datasets']] metadata['datasets'] = [transform_dataset(dataset) for dataset in metadata['datasets']]
# transform references
metadata['references'] = [transform_reference(reference) for reference in metadata['references']]
upload_data.calcs[i] = json.dumps(calc_data) upload_data.calcs[i] = json.dumps(calc_data)
return upload_data return upload_data
......
...@@ -70,7 +70,7 @@ class User(metainfo.MSection): ...@@ -70,7 +70,7 @@ class User(metainfo.MSection):
def repo_users() -> Dict[str, 'User']: def repo_users() -> Dict[str, 'User']:
from nomad import infrastructure from nomad import infrastructure
return { return {
user.repo_user_id: user str(user.repo_user_id): user
for user in infrastructure.keycloak.search_user() for user in infrastructure.keycloak.search_user()
if user.repo_user_id is not None if user.repo_user_id is not None
} }
......
...@@ -270,17 +270,18 @@ class Keycloak(): ...@@ -270,17 +270,18 @@ class Keycloak():
kwargs = {key: value[0] for key, value in keycloak_user.get('attributes', {}).items()} kwargs = {key: value[0] for key, value in keycloak_user.get('attributes', {}).items()}
return datamodel.User( return datamodel.User(
user_id=keycloak_user['id'], user_id=keycloak_user['id'],
email=keycloak_user['email'], email=keycloak_user.get('email'),
username=keycloak_user.get('username', None), username=keycloak_user.get('username'),
first_name=keycloak_user.get('firstName', None), first_name=keycloak_user.get('firstName'),
last_name=keycloak_user.get('lastName', None), last_name=keycloak_user.get('lastName'),
created=datetime.fromtimestamp(keycloak_user['createdTimestamp'] / 1000), created=datetime.fromtimestamp(keycloak_user['createdTimestamp'] / 1000),
**kwargs) **kwargs)
def search_user(self, query: str = None): def search_user(self, query: str = None, **kwargs):
kwargs = {}
if query is not None: if query is not None:
kwargs['query'] = dict(search=query) kwargs['query'] = dict(search=query)
else:
kwargs['query'] = dict(max=1000)
try: try:
keycloak_results = self._admin_client.get_users(**kwargs) keycloak_results = self._admin_client.get_users(**kwargs)
except Exception as e: except Exception as e:
......
...@@ -805,7 +805,7 @@ class MSection(metaclass=MObjectMeta): ...@@ -805,7 +805,7 @@ class MSection(metaclass=MObjectMeta):
return bool(value) return bool(value)
else: else:
if type(value) != quantity_def.type: if value is not None and type(value) != quantity_def.type:
raise TypeError( raise TypeError(
'The value %s with type %s for quantity %s is not of type %s' % 'The value %s with type %s for quantity %s is not of type %s' %
(value, type(value), quantity_def, quantity_def.type)) (value, type(value), quantity_def, quantity_def.type))
......
export NOMAD_CLIENT_URL=https://labdev-nomad.esc.rzg.mpg.de/fairdi/nomad/testing-major/api
export NOMAD_CLIENT_USER=admin
export NOMAD_FS_LOCAL_TMP=/nomad/fairdi/testing_major/fs/tmp
export NOMAD_MONGO_DB_NAME=fairdi_nomad_testing_major
export NOMAD_ELASTIC_INDEX_NAME=fairdi_nomad_testing_major
export NOMAD_FS_LOCAL_TMP=/nomad/fairdi/testing_major/fs/tmp
export NOMAD_FS_STAGING=/nomad/fairdi/testing_major/fs/staging
export NOMAD_FS_PUBLIC=/nomad/fairdi/testing_major/fs/public
export NOMAD_FS_TMP=/nomad/fairdi/testing_major/fs/tmp
export NOMAD_SERVICE_ADMIN_USER_ID="82efac55-6187-408c-8027-b98580c0e1c5"
export NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod
export NOMAD_FS_PREFIX_SIZE=1
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment