From b27e6a7bb1c0cf4fa6cd85aa471302ee5dc13cab Mon Sep 17 00:00:00 2001 From: Markus Scheidgen <markus.scheidgen@gmail.com> Date: Wed, 13 Nov 2019 14:59:46 +0100 Subject: [PATCH] Fixes to migration in cli mirror command. --- nomad/cli/client/mirror.py | 10 ++++++++-- nomad/datamodel/metainfo.py | 2 +- nomad/infrastructure.py | 13 +++++++------ nomad/metainfo/metainfo.py | 2 +- ops/deployments/nomad.testing-major.env.sh | 12 ++++++++++++ 5 files changed, 29 insertions(+), 10 deletions(-) create mode 100644 ops/deployments/nomad.testing-major.env.sh diff --git a/nomad/cli/client/mirror.py b/nomad/cli/client/mirror.py index a3a3c0ab63..2ea586ba1f 100644 --- a/nomad/cli/client/mirror.py +++ b/nomad/cli/client/mirror.py @@ -37,7 +37,7 @@ def v0Dot6(upload_data): """ Inplace transforms v0.6.x upload data into v0.7.x upload data. """ def tarnsform_user_id(source_user_id): - target_user = User.repo_users.get(source_user_id) + target_user = User.repo_users().get(str(source_user_id)) if target_user is None: __logger.error('user does not exist in target', source_user_id=source_user_id) raise KeyError @@ -61,11 +61,14 @@ def v0Dot6(upload_data): return target_dataset.dataset_id + def transform_reference(reference): + return reference['value'] + upload = json.loads(upload_data.upload) upload['user_id'] = tarnsform_user_id(upload['user_id']) upload_data.upload = json.dumps(upload) - for calc_data_json, i in enumerate(upload_data.calcs): + for i, calc_data_json in enumerate(upload_data.calcs): calc_data = json.loads(calc_data_json) metadata = calc_data['metadata'] @@ -77,6 +80,9 @@ def v0Dot6(upload_data): # transform datasets metadata['datasets'] = [transform_dataset(dataset) for dataset in metadata['datasets']] + # transform references + metadata['references'] = [transform_reference(reference) for reference in metadata['references']] + upload_data.calcs[i] = json.dumps(calc_data) return upload_data diff --git a/nomad/datamodel/metainfo.py b/nomad/datamodel/metainfo.py index ac49269626..0a6c3ef5dd 100644 --- a/nomad/datamodel/metainfo.py +++ b/nomad/datamodel/metainfo.py @@ -70,7 +70,7 @@ class User(metainfo.MSection): def repo_users() -> Dict[str, 'User']: from nomad import infrastructure return { - user.repo_user_id: user + str(user.repo_user_id): user for user in infrastructure.keycloak.search_user() if user.repo_user_id is not None } diff --git a/nomad/infrastructure.py b/nomad/infrastructure.py index 51cecb4c39..6bc273ac7d 100644 --- a/nomad/infrastructure.py +++ b/nomad/infrastructure.py @@ -270,17 +270,18 @@ class Keycloak(): kwargs = {key: value[0] for key, value in keycloak_user.get('attributes', {}).items()} return datamodel.User( user_id=keycloak_user['id'], - email=keycloak_user['email'], - username=keycloak_user.get('username', None), - first_name=keycloak_user.get('firstName', None), - last_name=keycloak_user.get('lastName', None), + email=keycloak_user.get('email'), + username=keycloak_user.get('username'), + first_name=keycloak_user.get('firstName'), + last_name=keycloak_user.get('lastName'), created=datetime.fromtimestamp(keycloak_user['createdTimestamp'] / 1000), **kwargs) - def search_user(self, query: str = None): - kwargs = {} + def search_user(self, query: str = None, **kwargs): if query is not None: kwargs['query'] = dict(search=query) + else: + kwargs['query'] = dict(max=1000) try: keycloak_results = self._admin_client.get_users(**kwargs) except Exception as e: diff --git a/nomad/metainfo/metainfo.py b/nomad/metainfo/metainfo.py index c89d090785..608cb01533 100644 --- a/nomad/metainfo/metainfo.py +++ b/nomad/metainfo/metainfo.py @@ -805,7 +805,7 @@ class MSection(metaclass=MObjectMeta): return bool(value) else: - if type(value) != quantity_def.type: + if value is not None and type(value) != quantity_def.type: raise TypeError( 'The value %s with type %s for quantity %s is not of type %s' % (value, type(value), quantity_def, quantity_def.type)) diff --git a/ops/deployments/nomad.testing-major.env.sh b/ops/deployments/nomad.testing-major.env.sh new file mode 100644 index 0000000000..17b8777ae9 --- /dev/null +++ b/ops/deployments/nomad.testing-major.env.sh @@ -0,0 +1,12 @@ +export NOMAD_CLIENT_URL=https://labdev-nomad.esc.rzg.mpg.de/fairdi/nomad/testing-major/api +export NOMAD_CLIENT_USER=admin +export NOMAD_FS_LOCAL_TMP=/nomad/fairdi/testing_major/fs/tmp +export NOMAD_MONGO_DB_NAME=fairdi_nomad_testing_major +export NOMAD_ELASTIC_INDEX_NAME=fairdi_nomad_testing_major +export NOMAD_FS_LOCAL_TMP=/nomad/fairdi/testing_major/fs/tmp +export NOMAD_FS_STAGING=/nomad/fairdi/testing_major/fs/staging +export NOMAD_FS_PUBLIC=/nomad/fairdi/testing_major/fs/public +export NOMAD_FS_TMP=/nomad/fairdi/testing_major/fs/tmp +export NOMAD_SERVICE_ADMIN_USER_ID="82efac55-6187-408c-8027-b98580c0e1c5" +export NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod +export NOMAD_FS_PREFIX_SIZE=1 -- GitLab