Commit 4ff5585b authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Migration related fixes. New envrionment.

parent c7fb69e5
Pipeline #52318 passed with stages
in 19 minutes and 37 seconds
......@@ -391,6 +391,8 @@ class Package(Document):
""" The report of the last successful migration of this package """
skip_migration = BooleanField()
""" Packages with known problems can be marked to be not migrated """
skip_migration_reason = StringField()
""" Optional description of the reason for skipping migration """
migration_failure = StringField()
""" String that describe the cause for last failed migration attempt """
......@@ -1265,6 +1267,10 @@ class NomadCOEMigration:
overall_report.total_packages += 1
overall_report.skipped_packages += 1
package_report = package.report
if package_report is None:
package_report = Report()
elif package.migration_version is not None and package.migration_version >= self.migration_version:
if only_republish:
self.republish_package(package)
......
export NOMAD_CLIENT_URL=https://labdev-nomad.esc.rzg.mpg.de/fairdi/nomad/testing/api
export NOMAD_CLIENT_USER=admin
export NOMAD_FS_LOCAL_TMP=/nomad/fairdi/testing/fs/tmp
export NOMAD_MONGO_DB_NAME=fairdi_nomad_testing
export NOMAD_ELASTIC_INDEX_NAME=fairdi_nomad_testing
export NOMAD_FS_LOCAL_TMP=/nomad/fairdi/testing/fs/tmp
export NOMAD_FS_STAGING=/nomad/fairdi/testing/fs/staging
export NOMAD_FS_PUBLIC=/nomad/fairdi/testing/fs/public
export NOMAD_FS_TMP=/nomad/fairdi/testing/fs/tmp
proxy:
nodePort: 30007
external:
host: "labdev-nomad.esc.rzg.mpg.de"
path: "/fairdi/nomad/testing"
api:
disableReset: false
worker:
replicas: 1
routing: "queue"
dbname: fairdi_nomad_testing
postgres:
publish_enabled: false
uploadurl: 'https://labdev-nomad.rzg.mpg.de/fairdi/nomad/testing/upload'
volumes:
prefixSize: 2
public: /nomad/fairdi/testing/fs/public
staging: /nomad/fairdi/testing/fs/staging
tmp: /nomad/fairdi/testing/fs/tmp
nomad: /nomad
......@@ -19,12 +19,13 @@ def retrieve_remote_data():
pid = calc['metadata']['pid']
upload = calc['upload_id']
calcs = pid_dict.get(pid)
if calcs is None:
calcs = []
pid_dict[pid] = calcs
pid_calcs = pid_dict.get(pid)
if pid_calcs is None:
pid_calcs = []
pid_dict[pid] = pid_calcs
calcs.append(upload)
if upload != 'j-IVBd3MQWWvTJ6pg5oaSw':
pid_calcs.append(upload)
count += 1
if count % 100000 == 0:
......@@ -41,8 +42,11 @@ def load_local_data():
return json.load(f)
pid_dict = load_local_data()
# pid_dict = retrieve_remote_data()
try:
pid_dict = load_local_data()
except Exception:
pid_dict = retrieve_remote_data()
print('data available ...')
......
from nomad import infrastructure
from dateutil.parser import parse
infrastructure.setup_logging()
mongo = infrastructure.setup_mongo()
......@@ -9,8 +10,14 @@ uploads = mongo.fairdi_nomad_migration.upload
def doit(upload):
example = calcs.find_one({'upload_id': upload})
user_id = example['metadata']['uploader']['id']
upload_time = example['metadata']['upload_time']
uploads.update_one({'_id': upload}, {'$set': {'user_id': user_id}})
uploads.update_one(
{'_id': upload},
{'$set': {
'user_id': str(user_id),
'upload_time': parse(str(upload_time))
}})
for upload in uploads.distinct('_id'):
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment