Commit 3525ab34 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added a migration tutorial and command #395. Minor fixes.

parent e3276dd8
Pipeline #80494 passed with stages
in 48 minutes and 28 seconds
......@@ -122,7 +122,7 @@ function BetaSnack() {
const [understood, setUnderstood] = useState(false)
if (!version) {
console.log.warning('no version data available')
console.warn('no version data available')
return ''
}
......
......@@ -13,8 +13,8 @@ export const keycloakBase = window.nomadEnv.keycloakBase
export const keycloakRealm = window.nomadEnv.keycloakRealm
export const keycloakClientId = window.nomadEnv.keycloakClientId
export const debug = window.nomadEnv.debug || false
export const matomoEnabled = window.nomadEnv.matomoEnabled
export const encyclopediaEnabled = window.nomadEnv.encyclopediaEnabled
export const matomoEnabled = window.nomadEnv.matomoEnabled || false
export const encyclopediaEnabled = window.nomadEnv.encyclopediaEnabled || false
export const email = 'support@nomad-lab.eu'
export const maxLogsToShow = 50
......
......@@ -17,15 +17,17 @@ import click
from .admin import admin
@admin.group(help='Migrate data from older NOMAD versions')
@admin.command(help='Migrate data from older NOMAD versions')
@click.option('--mongo-db', help='The database name of the existing data', type=str)
def migrate(mongo_db: str, elastic_index: str):
def migrate(mongo_db: str):
import pymongo
import sys
import json
from nomad import config, processing as proc, doi as nomad_doi, datamodel
from nomad import config, processing as proc, doi as nomad_doi, datamodel, infrastructure
from nomad.app.api.mirror import _upload_data
from nomad.cli.client.mirror import v0Dot7, fix_time, _Dataset
from bson.json_util import dumps
infrastructure.setup()
_Dataset = datamodel.Dataset.m_def.a_mongo.mongo_cls
......@@ -35,15 +37,13 @@ def migrate(mongo_db: str, elastic_index: str):
print('The given mongo database %s does not exist' % mongo_db)
sys.exit(1)
for upload in db.uploads.find():
print('There are %d uploads in the source database' % db.upload.find().count())
for upload in db.upload.find():
print('migrating upload with id %s' % upload['_id'])
upload_data = _upload_data(upload['_id'], json.dumps(upload), calcs_col=db.calcs, datasets_col=db.datasets, dois_col=db.d_o_i)
upload_json = dumps(upload)
upload_data = _upload_data(upload['_id'], upload_json, calcs_col=db.calc, datasets_col=db.dataset, dois_col=db.d_o_i)
upload_data = v0Dot7(upload_data)
proc.Upload._get_collection().insert(upload)
for calc in db.calcs.find(dict(upload_id=upload['_id'])):
proc.Upload.from_dict(upload).save()
# create mongo
try:
upload = proc.Upload.from_json(upload_data['upload'], created=True)
......
......@@ -378,7 +378,7 @@ class StagingUploadFiles(UploadFiles):
target_dir: Overwrite the used directory to extract to. Default is the raw directory of this upload.
'''
assert not self.is_frozen
assert os.path.exists(path)
assert os.path.exists(path), '%s does not exist' % path
self._size += os.stat(path).st_size
target_dir = self._raw_dir if target_dir is None else target_dir
if prefix is not None:
......
......@@ -104,7 +104,7 @@ services:
# nomad worker (processing)
worker:
restart: always
image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest
image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:stable
container_name: nomad_oasis_worker
environment:
<<: *nomad_backend_env
......@@ -121,7 +121,7 @@ services:
# nomad app (api + gui)
app:
restart: always
image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest
image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:stable
container_name: nomad_oasis_app
environment:
<<: *nomad_backend_env
......@@ -187,6 +187,12 @@ keycloak:
username: '<your admin username>'
password: '<your admin user password>'
oasis: true
mongo:
db_name: nomad_v0_8
elastic:
index_name: nomad_v0_8
```
You need to change:
......@@ -206,7 +212,7 @@ window.nomadEnv = {
'keycloakBase': 'https://nomad-lab.eu/fairdi/keycloak/auth/',
'keycloakRealm': 'fairdi_nomad_prod',
'keycloakClientId': 'nomad_public',
'debug': false,
'debug': false
};
```
......@@ -320,6 +326,49 @@ If you want to report problems with your OASIS. Please provide the logs for
- nomad_oasis_worker
- nomad_oasis_gui
## Migrating from an older version (0.7.x to 0.8.x)
Between versions 0.7.x and 0.8.x we needed to change how archive and metadata data is stored
internally in files and databases. This means you cannot simply start a new version of
NOMAD on top of the old data. But there is a strategy to adapt the data.
First, shutdown the OASIS and remove all old container.
```
docker-compose stop
docker-compose rm -f
```
Update you config files (`docker-compose.yaml`, `nomad.yaml`, `env.js`, `nginx.conf`) according
to the latest documentation (see above). Especially make sure to select a new name for
databases and search index in your `nomad.yaml` (e.g. `nomad_v0_8`). This will
allow us to create new data while retaining the old, i.e. to copy the old data over.
Make sure you get the latest images and start the OASIS with the new version of NOMAD:
```
docker-compose pull
docker-compose up -d
```
If you go to the GUI of your OASIS, it should now show the new version and appear empty,
because we are using a different database and search index now.
To migrate the data, we created a command that you can run within your OASIS' NOMAD
application container. This command takes the old database name as argument, it will copy
all data over and then reprocess the data to create data in the new archive format and
populate the search index. The default database name in version 0.7.x installations was `nomad_fairdi`.
```
docker exec nomad_oasis_app -- nomad admin migrate --mongo-db nomad_fairdi
```
Now all your data should appear in your OASIS again. If you like, you can remove the
old index and database:
```
docker exec nomad_oasis_app bash -c 'curl -X DELETE http://elastic:9200/nomad_fairdi'
docker exec nomad_oasis_mongo bash -c 'mongo nomad_fairdi --eval "printjson(db.dropDatabase())"'
```
## NOMAD Oasis FAQ
### Why use an Oasis?
......
......@@ -18,6 +18,8 @@ x-common-variables: &nomad_backend_env
NOMAD_RABBITMQ_HOST: rabbitmq
NOMAD_ELASTIC_HOST: elastic
NOMAD_MONGO_HOST: mongo
NOMAD_MONGO_DB_NAME: nomad_v0_8
NOMAD_ELASTIC_INDEX_NAME: nomad_v0_8
services:
# broker for celery
......@@ -56,7 +58,7 @@ services:
# nomad worker (processing)
worker:
restart: always
image: nomad/local:latest
image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:stable
container_name: nomad_oasis_worker
environment:
<<: *nomad_backend_env
......@@ -66,14 +68,13 @@ services:
- elastic
- mongo
volumes:
- nomad_oasis_files:/app/.volumes/fs
- ./nomad.yaml:/app/nomad.yaml
command: python -m celery worker -l info -A nomad.processing -Q celery,calcs,uploads
# nomad app (api + gui)
app:
restart: always
image: nomad/local:latest
image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:stable
container_name: nomad_oasis_app
environment:
<<: *nomad_backend_env
......@@ -83,7 +84,6 @@ services:
- elastic
- mongo
volumes:
- nomad_oasis_files:/app/.volumes/fs
- ./nomad.yaml:/app/nomad.yaml
- ./env.js:/app/gui/build/env.js
- ./gunicorn.log.conf:/app/gunicorn.log.conf
......
......@@ -4,4 +4,6 @@ window.nomadEnv = {
'keycloakRealm': 'fairdi_nomad_prod',
'keycloakClientId': 'nomad_public',
'debug': false,
'matomoEnabled': false,
'encyclopediaEnabled': false
};
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment