diff --git a/gui/src/components/App.js b/gui/src/components/App.js index cda7598dd27b346d4eb5384f1ae67c8dd921ce89..8724924be581fa03b6839784c99306061ab5899c 100644 --- a/gui/src/components/App.js +++ b/gui/src/components/App.js @@ -122,7 +122,7 @@ function BetaSnack() { const [understood, setUnderstood] = useState(false) if (!version) { - console.log.warning('no version data available') + console.warn('no version data available') return '' } diff --git a/gui/src/config.js b/gui/src/config.js index e53e0ab5538966b767a410297c3c60ce095c7d86..bb17a8ac788660ab3d9ca6061a249fbfacfe25eb 100644 --- a/gui/src/config.js +++ b/gui/src/config.js @@ -13,8 +13,8 @@ export const keycloakBase = window.nomadEnv.keycloakBase export const keycloakRealm = window.nomadEnv.keycloakRealm export const keycloakClientId = window.nomadEnv.keycloakClientId export const debug = window.nomadEnv.debug || false -export const matomoEnabled = window.nomadEnv.matomoEnabled -export const encyclopediaEnabled = window.nomadEnv.encyclopediaEnabled +export const matomoEnabled = window.nomadEnv.matomoEnabled || false +export const encyclopediaEnabled = window.nomadEnv.encyclopediaEnabled || false export const email = 'support@nomad-lab.eu' export const maxLogsToShow = 50 diff --git a/nomad/cli/admin/migrate.py b/nomad/cli/admin/migrate.py index 95ea0b5484b3637b4964f9c73499db2144ffa2ba..869f2c43b1c0201f40866ca294b7eda3ada5a839 100644 --- a/nomad/cli/admin/migrate.py +++ b/nomad/cli/admin/migrate.py @@ -17,15 +17,17 @@ import click from .admin import admin -@admin.group(help='Migrate data from older NOMAD versions') +@admin.command(help='Migrate data from older NOMAD versions') @click.option('--mongo-db', help='The database name of the existing data', type=str) -def migrate(mongo_db: str, elastic_index: str): +def migrate(mongo_db: str): import pymongo import sys - import json - from nomad import config, processing as proc, doi as nomad_doi, datamodel + from nomad import config, processing as proc, doi as nomad_doi, datamodel, infrastructure from nomad.app.api.mirror import _upload_data from nomad.cli.client.mirror import v0Dot7, fix_time, _Dataset + from bson.json_util import dumps + + infrastructure.setup() _Dataset = datamodel.Dataset.m_def.a_mongo.mongo_cls @@ -35,15 +37,13 @@ def migrate(mongo_db: str, elastic_index: str): print('The given mongo database %s does not exist' % mongo_db) sys.exit(1) - for upload in db.uploads.find(): + print('There are %d uploads in the source database' % db.upload.find().count()) + for upload in db.upload.find(): print('migrating upload with id %s' % upload['_id']) - upload_data = _upload_data(upload['_id'], json.dumps(upload), calcs_col=db.calcs, datasets_col=db.datasets, dois_col=db.d_o_i) + upload_json = dumps(upload) + upload_data = _upload_data(upload['_id'], upload_json, calcs_col=db.calc, datasets_col=db.dataset, dois_col=db.d_o_i) upload_data = v0Dot7(upload_data) - proc.Upload._get_collection().insert(upload) - for calc in db.calcs.find(dict(upload_id=upload['_id'])): - proc.Upload.from_dict(upload).save() - # create mongo try: upload = proc.Upload.from_json(upload_data['upload'], created=True) diff --git a/nomad/files.py b/nomad/files.py index 3f1d597af760fc1fb3438ea7a7fdf787b30700c0..cfbee7081a77dc31f1fdb97650edad5939afd93f 100644 --- a/nomad/files.py +++ b/nomad/files.py @@ -378,7 +378,7 @@ class StagingUploadFiles(UploadFiles): target_dir: Overwrite the used directory to extract to. Default is the raw directory of this upload. ''' assert not self.is_frozen - assert os.path.exists(path) + assert os.path.exists(path), '%s does not exist' % path self._size += os.stat(path).st_size target_dir = self._raw_dir if target_dir is None else target_dir if prefix is not None: diff --git a/ops/docker-compose/nomad-oasis/README.md b/ops/docker-compose/nomad-oasis/README.md index 75b87ad91bfe8c5603ceb1d85aa0c18caad2834b..52fe9632b8b1e995891f50801d2713ff34743e30 100644 --- a/ops/docker-compose/nomad-oasis/README.md +++ b/ops/docker-compose/nomad-oasis/README.md @@ -104,7 +104,7 @@ services: # nomad worker (processing) worker: restart: always - image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest + image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:stable container_name: nomad_oasis_worker environment: <<: *nomad_backend_env @@ -121,7 +121,7 @@ services: # nomad app (api + gui) app: restart: always - image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest + image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:stable container_name: nomad_oasis_app environment: <<: *nomad_backend_env @@ -187,6 +187,12 @@ keycloak: username: '<your admin username>' password: '<your admin user password>' oasis: true + +mongo: + db_name: nomad_v0_8 + +elastic: + index_name: nomad_v0_8 ``` You need to change: @@ -206,7 +212,7 @@ window.nomadEnv = { 'keycloakBase': 'https://nomad-lab.eu/fairdi/keycloak/auth/', 'keycloakRealm': 'fairdi_nomad_prod', 'keycloakClientId': 'nomad_public', - 'debug': false, + 'debug': false }; ``` @@ -320,6 +326,49 @@ If you want to report problems with your OASIS. Please provide the logs for - nomad_oasis_worker - nomad_oasis_gui +## Migrating from an older version (0.7.x to 0.8.x) + +Between versions 0.7.x and 0.8.x we needed to change how archive and metadata data is stored +internally in files and databases. This means you cannot simply start a new version of +NOMAD on top of the old data. But there is a strategy to adapt the data. + +First, shutdown the OASIS and remove all old container. +``` +docker-compose stop +docker-compose rm -f +``` + +Update you config files (`docker-compose.yaml`, `nomad.yaml`, `env.js`, `nginx.conf`) according +to the latest documentation (see above). Especially make sure to select a new name for +databases and search index in your `nomad.yaml` (e.g. `nomad_v0_8`). This will +allow us to create new data while retaining the old, i.e. to copy the old data over. + +Make sure you get the latest images and start the OASIS with the new version of NOMAD: +``` +docker-compose pull +docker-compose up -d +``` + +If you go to the GUI of your OASIS, it should now show the new version and appear empty, +because we are using a different database and search index now. + +To migrate the data, we created a command that you can run within your OASIS' NOMAD +application container. This command takes the old database name as argument, it will copy +all data over and then reprocess the data to create data in the new archive format and +populate the search index. The default database name in version 0.7.x installations was `nomad_fairdi`. + +``` +docker exec nomad_oasis_app -- nomad admin migrate --mongo-db nomad_fairdi +``` + +Now all your data should appear in your OASIS again. If you like, you can remove the +old index and database: + +``` +docker exec nomad_oasis_app bash -c 'curl -X DELETE http://elastic:9200/nomad_fairdi' +docker exec nomad_oasis_mongo bash -c 'mongo nomad_fairdi --eval "printjson(db.dropDatabase())"' +``` + ## NOMAD Oasis FAQ ### Why use an Oasis? diff --git a/ops/docker-compose/nomad-oasis/docker-compose.yml b/ops/docker-compose/nomad-oasis/docker-compose.yml index c575238a3f8b322dfb287fc87f863407c1194a67..090ffbf44224480d9f7acf460a322e6964c141d1 100644 --- a/ops/docker-compose/nomad-oasis/docker-compose.yml +++ b/ops/docker-compose/nomad-oasis/docker-compose.yml @@ -18,6 +18,8 @@ x-common-variables: &nomad_backend_env NOMAD_RABBITMQ_HOST: rabbitmq NOMAD_ELASTIC_HOST: elastic NOMAD_MONGO_HOST: mongo + NOMAD_MONGO_DB_NAME: nomad_v0_8 + NOMAD_ELASTIC_INDEX_NAME: nomad_v0_8 services: # broker for celery @@ -56,7 +58,7 @@ services: # nomad worker (processing) worker: restart: always - image: nomad/local:latest + image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:stable container_name: nomad_oasis_worker environment: <<: *nomad_backend_env @@ -66,14 +68,13 @@ services: - elastic - mongo volumes: - - nomad_oasis_files:/app/.volumes/fs - ./nomad.yaml:/app/nomad.yaml command: python -m celery worker -l info -A nomad.processing -Q celery,calcs,uploads # nomad app (api + gui) app: restart: always - image: nomad/local:latest + image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:stable container_name: nomad_oasis_app environment: <<: *nomad_backend_env @@ -83,7 +84,6 @@ services: - elastic - mongo volumes: - - nomad_oasis_files:/app/.volumes/fs - ./nomad.yaml:/app/nomad.yaml - ./env.js:/app/gui/build/env.js - ./gunicorn.log.conf:/app/gunicorn.log.conf diff --git a/ops/docker-compose/nomad-oasis/env.js b/ops/docker-compose/nomad-oasis/env.js index d4fd251fd81d3535a9e280c7fbfd6bd8645b41f1..6115af96a176366288335647e6aa162197e61512 100644 --- a/ops/docker-compose/nomad-oasis/env.js +++ b/ops/docker-compose/nomad-oasis/env.js @@ -4,4 +4,6 @@ window.nomadEnv = { 'keycloakRealm': 'fairdi_nomad_prod', 'keycloakClientId': 'nomad_public', 'debug': false, + 'matomoEnabled': false, + 'encyclopediaEnabled': false }; \ No newline at end of file