diff --git a/nomad/admin/__init__.py b/nomad/admin/__init__.py
index bc1477afadd3624e6fb1934279ec0f9f02dd93f5..2425b6f3082932780cdca55f1ed238f624f97aa8 100644
--- a/nomad/admin/__init__.py
+++ b/nomad/admin/__init__.py
@@ -19,8 +19,8 @@ Swagger/bravado based python client library for the API and various usefull shel
from nomad.utils import POPO
from . import upload, run
-from .__main__ import cli as cli_main
+from .cli import cli
-def cli():
- cli_main(obj=POPO()) # pylint: disable=E1120,E1123
+def run_cli():
+ cli(obj=POPO()) # pylint: disable=E1120,E1123
diff --git a/nomad/admin/__main__.py b/nomad/admin/__main__.py
index 240b10cdd93641f13560a7f5558b7a1f8c249b6e..df3ff3a63be7fea727fcbfca9d144e9fd9135500 100644
--- a/nomad/admin/__main__.py
+++ b/nomad/admin/__main__.py
@@ -12,137 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import click
-import logging
-import os
-import sys
-import shutil
-from tabulate import tabulate
-from elasticsearch_dsl import A
-
-from nomad import config as nomad_config, infrastructure, processing
-from nomad.search import Search
-
-
-@click.group(help='''The nomad admin command to do nasty stuff directly on the databases.
- Remember: With great power comes great responsibility!''')
-@click.option('-v', '--verbose', help='sets log level to info', is_flag=True)
-@click.option('--debug', help='sets log level to debug', is_flag=True)
-@click.option('--config', help='the config file to use')
-@click.pass_context
-def cli(ctx, verbose: bool, debug: bool, config: str):
- if config is not None:
- nomad_config.load_config(config_file=config)
-
- if debug:
- nomad_config.console_log_level = logging.DEBUG
- elif verbose:
- nomad_config.console_log_level = logging.INFO
- else:
- nomad_config.console_log_level = logging.WARNING
-
- nomad_config.service = os.environ.get('NOMAD_SERVICE', 'admin')
- infrastructure.setup_logging()
-
-
-@cli.command(help='Runs tests and linting. Useful before commit code.')
-@click.option('--skip-tests', help='Do not test, just do code checks.', is_flag=True)
-def qa(skip_tests: bool):
- os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
- ret_code = 0
- if not skip_tests:
- click.echo('Run tests ...')
- ret_code += os.system('python -m pytest -svx tests')
- click.echo('Run code style checks ...')
- ret_code += os.system('python -m pycodestyle --ignore=E501,E701 nomad tests')
- click.echo('Run linter ...')
- ret_code += os.system('python -m pylint --load-plugins=pylint_mongoengine nomad tests')
- click.echo('Run static type checks ...')
- ret_code += os.system('python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests')
-
- sys.exit(ret_code)
-
-
-@cli.command(help='Checks consistency of files and es vs mongo and deletes orphan entries.')
-@click.option('--dry', is_flag=True, help='Do not delete anything, just check.')
-@click.option('--skip-calcs', is_flag=True, help='Skip cleaning calcs with missing uploads.')
-@click.option('--skip-fs', is_flag=True, help='Skip cleaning the filesystem.')
-@click.option('--skip-es', is_flag=True, help='Skip cleaning the es index.')
-def clean(dry, skip_calcs, skip_fs, skip_es):
- infrastructure.setup_logging()
- mongo_client = infrastructure.setup_mongo()
- infrastructure.setup_elastic()
-
- if not skip_calcs:
- uploads_for_calcs = mongo_client[nomad_config.mongo.db_name]['calc'].distinct('upload_id')
- uploads = {}
- for upload in mongo_client[nomad_config.mongo.db_name]['upload'].distinct('_id'):
- uploads[upload] = True
-
- missing_uploads = []
- for upload_for_calc in uploads_for_calcs:
- if upload_for_calc not in uploads:
- missing_uploads.append(upload_for_calc)
-
- if not dry and len(missing_uploads) > 0:
- input('Will delete calcs (mongo + es) for %d missing uploads. Press any key to continue ...' % len(missing_uploads))
-
- for upload in missing_uploads:
- mongo_client[nomad_config.mongo.db_name]['calc'].remove(dict(upload_id=upload))
- Search(index=nomad_config.elastic.index_name).query('term', upload_id=upload).delete()
- else:
- print('Found %s uploads that have calcs in mongo, but there is no upload entry.' % len(missing_uploads))
- print('List first 10:')
- for upload in missing_uploads[:10]:
- print(upload)
-
- if not skip_fs:
- upload_dirs = []
- for bucket in [nomad_config.fs.public, nomad_config.fs.staging]:
- for prefix in os.listdir(bucket):
- for upload in os.listdir(os.path.join(bucket, prefix)):
- upload_dirs.append((upload, os.path.join(bucket, prefix, upload)))
-
- to_delete = list(
- path for upload, path in upload_dirs
- if processing.Upload.objects(upload_id=upload).first() is None)
-
- if not dry and len(to_delete) > 0:
- input('Will delete %d upload directories. Press any key to continue ...' % len(to_delete))
-
- for path in to_delete:
- shutil.rmtree(path)
- else:
- print('Found %d upload directories with no upload in mongo.' % len(to_delete))
- print('List first 10:')
- for path in to_delete[:10]:
- print(path)
-
- if not skip_es:
- search = Search(index=nomad_config.elastic.index_name)
- search.aggs.bucket('uploads', A('terms', field='upload_id', size=12000))
- response = search.execute()
-
- to_delete = list(
- (bucket.key, bucket.doc_count)
- for bucket in response.aggregations.uploads.buckets
- if processing.Upload.objects(upload_id=bucket.key).first() is None)
-
- calcs = 0
- for _, upload_calcs in to_delete:
- calcs += upload_calcs
-
- if not dry and len(to_delete) > 0:
- input(
- 'Will delete %d calcs in %d uploads from ES. Press any key to continue ...' %
- (calcs, len(to_delete)))
- for upload, _ in to_delete:
- Search(index=nomad_config.elastic.index_name).query('term', upload_id=upload).delete()
- else:
- print('Found %d calcs in %d uploads from ES with no upload in mongo.' % (calcs, len(to_delete)))
- print('List first 10:')
- tabulate(to_delete, headers=['id', '#calcs'])
+from nomad.utils import POPO
+from .cli import cli
if __name__ == '__main__':
- cli(obj={}) # pylint: disable=E1120,E1123
+ print('#######################')
+ cli(obj=POPO()) # pylint: disable=E1120,E1123
diff --git a/nomad/admin/cli.py b/nomad/admin/cli.py
new file mode 100644
index 0000000000000000000000000000000000000000..29b312d0123113e3f4153545026d497576e1ab48
--- /dev/null
+++ b/nomad/admin/cli.py
@@ -0,0 +1,144 @@
+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+import logging
+import os
+import sys
+import shutil
+from tabulate import tabulate
+from elasticsearch_dsl import A
+
+from nomad import config as nomad_config, infrastructure, processing
+from nomad.search import Search
+
+
+@click.group(help='''The nomad admin command to do nasty stuff directly on the databases.
+ Remember: With great power comes great responsibility!''')
+@click.option('-v', '--verbose', help='sets log level to info', is_flag=True)
+@click.option('--debug', help='sets log level to debug', is_flag=True)
+@click.option('--config', help='the config file to use')
+@click.pass_context
+def cli(ctx, verbose: bool, debug: bool, config: str):
+ if config is not None:
+ nomad_config.load_config(config_file=config)
+
+ if debug:
+ nomad_config.console_log_level = logging.DEBUG
+ elif verbose:
+ nomad_config.console_log_level = logging.INFO
+ else:
+ nomad_config.console_log_level = logging.WARNING
+
+ nomad_config.service = os.environ.get('NOMAD_SERVICE', 'admin')
+ infrastructure.setup_logging()
+
+
+@cli.command(help='Runs tests and linting. Useful before commit code.')
+@click.option('--skip-tests', help='Do not test, just do code checks.', is_flag=True)
+def qa(skip_tests: bool):
+ os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
+ ret_code = 0
+ if not skip_tests:
+ click.echo('Run tests ...')
+ ret_code += os.system('python -m pytest -svx tests')
+ click.echo('Run code style checks ...')
+ ret_code += os.system('python -m pycodestyle --ignore=E501,E701 nomad tests')
+ click.echo('Run linter ...')
+ ret_code += os.system('python -m pylint --load-plugins=pylint_mongoengine nomad tests')
+ click.echo('Run static type checks ...')
+ ret_code += os.system('python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests')
+
+ sys.exit(ret_code)
+
+
+@cli.command(help='Checks consistency of files and es vs mongo and deletes orphan entries.')
+@click.option('--dry', is_flag=True, help='Do not delete anything, just check.')
+@click.option('--skip-calcs', is_flag=True, help='Skip cleaning calcs with missing uploads.')
+@click.option('--skip-fs', is_flag=True, help='Skip cleaning the filesystem.')
+@click.option('--skip-es', is_flag=True, help='Skip cleaning the es index.')
+def clean(dry, skip_calcs, skip_fs, skip_es):
+ infrastructure.setup_logging()
+ mongo_client = infrastructure.setup_mongo()
+ infrastructure.setup_elastic()
+
+ if not skip_calcs:
+ uploads_for_calcs = mongo_client[nomad_config.mongo.db_name]['calc'].distinct('upload_id')
+ uploads = {}
+ for upload in mongo_client[nomad_config.mongo.db_name]['upload'].distinct('_id'):
+ uploads[upload] = True
+
+ missing_uploads = []
+ for upload_for_calc in uploads_for_calcs:
+ if upload_for_calc not in uploads:
+ missing_uploads.append(upload_for_calc)
+
+ if not dry and len(missing_uploads) > 0:
+ input('Will delete calcs (mongo + es) for %d missing uploads. Press any key to continue ...' % len(missing_uploads))
+
+ for upload in missing_uploads:
+ mongo_client[nomad_config.mongo.db_name]['calc'].remove(dict(upload_id=upload))
+ Search(index=nomad_config.elastic.index_name).query('term', upload_id=upload).delete()
+ else:
+ print('Found %s uploads that have calcs in mongo, but there is no upload entry.' % len(missing_uploads))
+ print('List first 10:')
+ for upload in missing_uploads[:10]:
+ print(upload)
+
+ if not skip_fs:
+ upload_dirs = []
+ for bucket in [nomad_config.fs.public, nomad_config.fs.staging]:
+ for prefix in os.listdir(bucket):
+ for upload in os.listdir(os.path.join(bucket, prefix)):
+ upload_dirs.append((upload, os.path.join(bucket, prefix, upload)))
+
+ to_delete = list(
+ path for upload, path in upload_dirs
+ if processing.Upload.objects(upload_id=upload).first() is None)
+
+ if not dry and len(to_delete) > 0:
+ input('Will delete %d upload directories. Press any key to continue ...' % len(to_delete))
+
+ for path in to_delete:
+ shutil.rmtree(path)
+ else:
+ print('Found %d upload directories with no upload in mongo.' % len(to_delete))
+ print('List first 10:')
+ for path in to_delete[:10]:
+ print(path)
+
+ if not skip_es:
+ search = Search(index=nomad_config.elastic.index_name)
+ search.aggs.bucket('uploads', A('terms', field='upload_id', size=12000))
+ response = search.execute()
+
+ to_delete = list(
+ (bucket.key, bucket.doc_count)
+ for bucket in response.aggregations.uploads.buckets
+ if processing.Upload.objects(upload_id=bucket.key).first() is None)
+
+ calcs = 0
+ for _, upload_calcs in to_delete:
+ calcs += upload_calcs
+
+ if not dry and len(to_delete) > 0:
+ input(
+ 'Will delete %d calcs in %d uploads from ES. Press any key to continue ...' %
+ (calcs, len(to_delete)))
+ for upload, _ in to_delete:
+ Search(index=nomad_config.elastic.index_name).query('term', upload_id=upload).delete()
+ else:
+ print('Found %d calcs in %d uploads from ES with no upload in mongo.' % (calcs, len(to_delete)))
+ print('List first 10:')
+ tabulate(to_delete, headers=['id', '#calcs'])
diff --git a/nomad/admin/run.py b/nomad/admin/run.py
index 431f290da1e01604fee0be92554d65294e9e2ad9..ef9e7d0a4bf26089d14eb213915f7eda6d918624 100644
--- a/nomad/admin/run.py
+++ b/nomad/admin/run.py
@@ -17,7 +17,7 @@ import asyncio
from concurrent.futures import ProcessPoolExecutor
from nomad import config
-from nomad.admin.__main__ import cli
+from .cli import cli
@cli.group(help='Run a nomad service locally (outside docker).')
diff --git a/nomad/admin/upload.py b/nomad/admin/upload.py
index 637f7166acf00b2dbd5592a7a829f33d83db0951..aa87a7ab0585f5ac38f0d2c3b293fa3315bd6792 100644
--- a/nomad/admin/upload.py
+++ b/nomad/admin/upload.py
@@ -18,7 +18,7 @@ from mongoengine import Q
from pymongo import UpdateOne
from nomad import processing as proc, config, infrastructure, utils, search, files, coe_repo
-from .__main__ import cli
+from .cli import cli
@cli.group(help='Upload related commands')
@@ -42,7 +42,7 @@ def upload(ctx, user: str, staging: bool, processing: bool, outdated: bool):
if outdated:
uploads = proc.Calc._get_collection().distinct(
'upload_id',
- {'metadata.nomad_version': { '$ne': config.version}})
+ {'metadata.nomad_version': {'$ne': config.version}})
query &= Q(upload_id__in=uploads)
ctx.obj.query = query
@@ -156,13 +156,13 @@ def re_process(ctx, uploads):
logger = utils.get_logger(__name__)
print('%d uploads selected, re-processing ...' % uploads.count())
- def re_process_upload(upload: str):
+ def re_process_upload(upload):
logger.info('re-processing started', upload_id=upload.upload_id)
upload.re_process_upload()
upload.block_until_complete(interval=.1)
- logger.info('re-processing complete', upload_id=upload_id)
+ logger.info('re-processing complete', upload_id=upload.upload_id)
count = 0
for upload in uploads:
diff --git a/nomad/api/upload.py b/nomad/api/upload.py
index 0cd90684613ac939f098618922a9b545372a08ca..225cdd3a873869734f96b493c31a7f1e41ab5a49 100644
--- a/nomad/api/upload.py
+++ b/nomad/api/upload.py
@@ -445,7 +445,7 @@ class UploadResource(Resource):
@login_really_required
def post(self, upload_id):
"""
- Execute an upload operation. Available operations: ``publish``
+ Execute an upload operation. Available operations are ``publish`` and ``re-process``
Publish accepts further meta data that allows to provide coauthors, comments,
external references, etc. See the model for details. The fields that start with
@@ -453,6 +453,10 @@ class UploadResource(Resource):
Publish changes the visibility of the upload. Clients can specify the visibility
via meta data.
+
+ Re-process will re-process the upload and produce updated repository metadata and
+ archive. Only published uploads that are not processing at the moment are allowed.
+ Only for uploads where calculations have been processed with an older nomad version.
"""
try:
upload = Upload.get(upload_id)
@@ -489,8 +493,21 @@ class UploadResource(Resource):
abort(400, message='The upload is still/already processed')
return upload, 200
+ elif operation == 're-process':
+ if upload.tasks_running or not upload.published:
+ abort(400, message='Can only non processing, re-process published uploads')
+
+ if len(metadata) > 0:
+ abort(400, message='You can not provide metadata for re-processing')
+
+ if len(upload.outdated_calcs) == 0:
+ abort(400, message='You can only re-process uploads with at least one outdated calculation')
+
+ upload.re_process_upload()
+
+ return upload, 200
- abort(400, message='Unsuported operation %s.' % operation)
+ abort(400, message='Unsupported operation %s.' % operation)
upload_command_model = api.model('UploadCommand', {
diff --git a/nomad/client/__init__.py b/nomad/client/__init__.py
index 12266f4124735bb7aa25f664c8a8eb99d9a83fe4..7963bb0008b740e9a6cc6ed7370fafe6665ed425 100644
--- a/nomad/client/__init__.py
+++ b/nomad/client/__init__.py
@@ -17,5 +17,9 @@ Swagger/bravado based python client library for the API and various usefull shel
"""
from . import local, migration, upload, integrationtests, parse
-from .__main__ import cli, create_client
+from .main import cli, create_client
from .upload import stream_upload_with_client
+
+
+def run_cli():
+ cli() # pylint: disable=E1120
diff --git a/nomad/client/__main__.py b/nomad/client/__main__.py
index 4d137e784151e9e721f70f0348086b76fcf1b1ab..3ea324acd171167e1a7cb9d1f33aee9f8214a6e2 100644
--- a/nomad/client/__main__.py
+++ b/nomad/client/__main__.py
@@ -12,106 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import os
-import sys
-import requests
-import click
-import logging
-from bravado.requests_client import RequestsClient
-from bravado.client import SwaggerClient
-from urllib.parse import urlparse
-
-import nomad.client
-from nomad import config as nomad_config
-from nomad import utils, infrastructure
-
-
-def create_client():
- return _create_client()
-
-
-def _create_client(*args, **kwargs):
- return __create_client(*args, **kwargs)
-
-
-def __create_client(user: str = nomad_config.client.user, password: str = nomad_config.client.password, ssl_verify: bool = True):
- """ A factory method to create the client. """
- host = urlparse(nomad_config.client.url).netloc.split(':')[0]
-
- if not ssl_verify:
- import warnings
- warnings.filterwarnings("ignore")
-
- http_client = RequestsClient(ssl_verify=ssl_verify)
- if user is not None:
- http_client.set_basic_auth(host, user, password)
-
- client = SwaggerClient.from_url(
- '%s/swagger.json' % nomad_config.client.url,
- http_client=http_client)
-
- utils.get_logger(__name__).info('created bravado client', user=user)
-
- return client
-
-
-def handle_common_errors(func):
- def wrapper(*args, **kwargs):
- try:
- func(*args, **kwargs)
- except requests.exceptions.ConnectionError:
- click.echo(
- '\nCould not connect to nomad at %s. '
- 'Check connection and url.' % nomad_config.client.url)
- sys.exit(0)
- return wrapper
-
-
-@click.group()
-@click.option('-n', '--url', default=nomad_config.client.url, help='The URL where nomad is running, default is "%s".' % nomad_config.client.url)
-@click.option('-u', '--user', default=None, help='the user name to login, default is "%s" login.' % nomad_config.client.user)
-@click.option('-w', '--password', default=nomad_config.client.password, help='the password used to login.')
-@click.option('-v', '--verbose', help='sets log level to info', is_flag=True)
-@click.option('--no-ssl-verify', help='disables SSL verificaton when talking to nomad.', is_flag=True)
-@click.option('--debug', help='sets log level to debug', is_flag=True)
-@click.option('--config', help='the config file to use')
-def cli(url: str, verbose: bool, debug: bool, user: str, password: str, config: str, no_ssl_verify: bool):
- if config is not None:
- nomad_config.load_config(config_file=config)
-
- if debug:
- nomad_config.console_log_level = logging.DEBUG
- elif verbose:
- nomad_config.console_log_level = logging.INFO
- else:
- nomad_config.console_log_level = logging.WARNING
-
- nomad_config.service = os.environ.get('NOMAD_SERVICE', 'client')
- infrastructure.setup_logging()
-
- logger = utils.get_logger(__name__)
-
- logger.info('Used nomad is %s' % url)
- logger.info('Used user is %s' % user)
-
- nomad_config.client.url = url
-
- global _create_client
-
- def _create_client(*args, **kwargs): # pylint: disable=W0612
- if user is not None:
- logger.info('create client', user=user)
- return __create_client(user=user, password=password, ssl_verify=not no_ssl_verify)
- else:
- logger.info('create anonymous client')
- return __create_client(ssl_verify=not no_ssl_verify)
-
-
-@cli.command(help='Attempts to reset the nomad.')
-def reset():
- from .__main__ import create_client
- create_client().admin.exec_reset_command().response()
-
+from .main import cli
if __name__ == '__main__':
- nomad.client.cli() # pylint: disable=E1120
+ cli() # pylint: disable=E1120
diff --git a/nomad/client/integrationtests.py b/nomad/client/integrationtests.py
index e330009f59461b6f71b0441f9e35b67365f413a5..e0018f4fbb9d8041b5cadc974b9547ef2ecf39fb 100644
--- a/nomad/client/integrationtests.py
+++ b/nomad/client/integrationtests.py
@@ -19,7 +19,7 @@ as a final integration test.
import time
-from .__main__ import cli
+from .main import cli
example_file = 'tests/data/proc/examples_vasp.zip'
@@ -27,7 +27,7 @@ example_file = 'tests/data/proc/examples_vasp.zip'
@cli.command(help='Runs a few example operations as a test.')
def integrationtests():
- from .__main__ import create_client
+ from .main import create_client
client = create_client()
print('upload with multiple code data')
diff --git a/nomad/client/local.py b/nomad/client/local.py
index e6982bb427cd522c514101fc294d3c0b1c3bd439..3c99d6f85ac30c85efbe263e038024e6b0330191 100644
--- a/nomad/client/local.py
+++ b/nomad/client/local.py
@@ -28,7 +28,7 @@ from nomad.datamodel import CalcWithMetadata
from nomad.parsing import LocalBackend
from nomad.client.parse import parse, normalize, normalize_all
-from .__main__ import cli
+from .main import cli
class CalcProcReproduction:
@@ -58,7 +58,7 @@ class CalcProcReproduction:
self.mainfile = mainfile
self.parser = None
- from .__main__ import create_client
+ from .main import create_client
client = create_client()
if self.mainfile is None:
try:
diff --git a/nomad/client/main.py b/nomad/client/main.py
new file mode 100644
index 0000000000000000000000000000000000000000..b84087fdc2b171e629fc9e1406df130f7bca1b5c
--- /dev/null
+++ b/nomad/client/main.py
@@ -0,0 +1,112 @@
+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import requests
+import click
+import logging
+from bravado.requests_client import RequestsClient
+from bravado.client import SwaggerClient
+from urllib.parse import urlparse
+
+from nomad import config as nomad_config
+from nomad import utils, infrastructure
+
+
+def create_client():
+ return _create_client()
+
+
+def _create_client(*args, **kwargs):
+ return __create_client(*args, **kwargs)
+
+
+def __create_client(user: str = nomad_config.client.user, password: str = nomad_config.client.password, ssl_verify: bool = True):
+ """ A factory method to create the client. """
+ host = urlparse(nomad_config.client.url).netloc.split(':')[0]
+
+ if not ssl_verify:
+ import warnings
+ warnings.filterwarnings("ignore")
+
+ http_client = RequestsClient(ssl_verify=ssl_verify)
+ if user is not None:
+ http_client.set_basic_auth(host, user, password)
+
+ client = SwaggerClient.from_url(
+ '%s/swagger.json' % nomad_config.client.url,
+ http_client=http_client)
+
+ utils.get_logger(__name__).info('created bravado client', user=user)
+
+ return client
+
+
+def handle_common_errors(func):
+ def wrapper(*args, **kwargs):
+ try:
+ func(*args, **kwargs)
+ except requests.exceptions.ConnectionError:
+ click.echo(
+ '\nCould not connect to nomad at %s. '
+ 'Check connection and url.' % nomad_config.client.url)
+ sys.exit(0)
+ return wrapper
+
+
+@click.group()
+@click.option('-n', '--url', default=nomad_config.client.url, help='The URL where nomad is running, default is "%s".' % nomad_config.client.url)
+@click.option('-u', '--user', default=None, help='the user name to login, default is "%s" login.' % nomad_config.client.user)
+@click.option('-w', '--password', default=nomad_config.client.password, help='the password used to login.')
+@click.option('-v', '--verbose', help='sets log level to info', is_flag=True)
+@click.option('--no-ssl-verify', help='disables SSL verificaton when talking to nomad.', is_flag=True)
+@click.option('--debug', help='sets log level to debug', is_flag=True)
+@click.option('--config', help='the config file to use')
+def cli(url: str, verbose: bool, debug: bool, user: str, password: str, config: str, no_ssl_verify: bool):
+ if config is not None:
+ nomad_config.load_config(config_file=config)
+
+ if debug:
+ nomad_config.console_log_level = logging.DEBUG
+ elif verbose:
+ nomad_config.console_log_level = logging.INFO
+ else:
+ nomad_config.console_log_level = logging.WARNING
+
+ nomad_config.service = os.environ.get('NOMAD_SERVICE', 'client')
+ infrastructure.setup_logging()
+
+ logger = utils.get_logger(__name__)
+
+ logger.info('Used nomad is %s' % url)
+ logger.info('Used user is %s' % user)
+
+ nomad_config.client.url = url
+
+ global _create_client
+
+ def _create_client(*args, **kwargs): # pylint: disable=W0612
+ if user is not None:
+ logger.info('create client', user=user)
+ return __create_client(user=user, password=password, ssl_verify=not no_ssl_verify)
+ else:
+ logger.info('create anonymous client')
+ return __create_client(ssl_verify=not no_ssl_verify)
+
+
+@cli.command(help='Attempts to reset the nomad.')
+def reset():
+ from .main import create_client
+ create_client().admin.exec_reset_command().response()
diff --git a/nomad/client/migration.py b/nomad/client/migration.py
index 79dac17aa60e1afa2478fc6e231a2a55cc72bffa..6653117a613e04c7560e075b7d5848a3d1c326c7 100644
--- a/nomad/client/migration.py
+++ b/nomad/client/migration.py
@@ -26,7 +26,7 @@ import json
from nomad import config, infrastructure
from nomad.migration import NomadCOEMigration, SourceCalc, Package, missing_calcs_data
-from .__main__ import cli
+from .main import cli
def _Migration(**kwargs) -> NomadCOEMigration:
diff --git a/nomad/client/parse.py b/nomad/client/parse.py
index e917a10ef4f15090ba0de9d223516928a88081d9..395cbb1117edd30857fc9aec038d513f8b3b5df2 100644
--- a/nomad/client/parse.py
+++ b/nomad/client/parse.py
@@ -9,7 +9,7 @@ from nomad.parsing import LocalBackend, parser_dict, match_parser
from nomad.normalizing import normalizers
from nomad.datamodel import CalcWithMetadata
-from .__main__ import cli
+from .main import cli
def parse(
diff --git a/nomad/client/upload.py b/nomad/client/upload.py
index b40fd209cc481f20aec852a8d33e675079806d72..1b86a37fd846e2a1be22d5ae16a364e80013e8ad 100644
--- a/nomad/client/upload.py
+++ b/nomad/client/upload.py
@@ -22,7 +22,7 @@ import requests
from nomad import utils, config
from nomad.processing import FAILURE, SUCCESS
-from .__main__ import cli, create_client
+from .main import cli, create_client
def stream_upload_with_client(client, stream, name=None):
diff --git a/nomad/config.py b/nomad/config.py
index ac13f62664972494645952b0f699dff50e023432..b4f6bfe44ad7d1152f571785a9f640927ccdb73d 100644
--- a/nomad/config.py
+++ b/nomad/config.py
@@ -259,7 +259,7 @@ def load_config(config_file: str = os.environ.get('NOMAD_CONFIG', 'nomad.yaml'))
if os.path.exists(config_file):
with open(config_file, 'r') as stream:
try:
- config_data = yaml.load(stream)
+ config_data = yaml.load(stream, Loader=getattr(yaml, 'FullLoader'))
except yaml.YAMLError as e:
logger.error('cannot read nomad config', exc_info=e)
diff --git a/nomad/processing/data.py b/nomad/processing/data.py
index c0917368c6ca140598b7e4c46cb9e2b70913d366..d4089a035bf39acc943ad2d91f244dac5b420b15 100644
--- a/nomad/processing/data.py
+++ b/nomad/processing/data.py
@@ -613,6 +613,9 @@ class Upload(Proc):
"""
assert self.published
+ logger = self.get_logger()
+ logger.info('started to re-process')
+
self.reset()
# mock the steps of actual processing
self._continue_with('uploading')
@@ -861,6 +864,12 @@ class Upload(Proc):
query = Calc.objects(upload_id=self.upload_id)[start:end]
return query.order_by(order_by) if order_by is not None else query
+ @property
+ def outdated_calcs(self):
+ return Calc.objects(
+ upload_id=self.upload_id, tasks_status=SUCCESS,
+ metadata__nomad_version__ne=config.version)
+
@property
def calcs(self):
return Calc.objects(upload_id=self.upload_id, tasks_status=SUCCESS)
diff --git a/setup.py b/setup.py
index e4a77ef976666dcc5fcab443d84ae285d2fc978e..ae2b9b2e49dd86bedac0bf2259bde00fc0bd122d 100644
--- a/setup.py
+++ b/setup.py
@@ -18,6 +18,6 @@ setup(
install_requires=reqs,
entry_points='''
[console_scripts]
- nomad=nomad.client:cli
- admin=nomad.admin:cli
+ nomad=nomad.client:run_cli
+ admin=nomad.admin:run_cli
''')
diff --git a/tests/conftest.py b/tests/conftest.py
index d33f597478ba4ebaee88e09fbd8087e09ff340f2..4a9d754f18f755cd307aeeb917b511b2decd5f36 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -546,6 +546,22 @@ def non_empty_processed(non_empty_uploaded: Tuple[str, str], test_user: coe_repo
return test_processing.run_processing(non_empty_uploaded, test_user)
+@pytest.mark.timeout(config.tests.default_timeout)
+@pytest.fixture(scope='function')
+def published(non_empty_processed: processing.Upload, example_user_metadata) -> processing.Upload:
+ """
+ Provides a processed upload. Upload was uploaded with test_user.
+ """
+ non_empty_processed.compress_and_set_metadata(example_user_metadata)
+ non_empty_processed.publish_upload()
+ try:
+ non_empty_processed.block_until_complete(interval=.01)
+ except Exception:
+ pass
+
+ return non_empty_processed
+
+
@pytest.fixture(scope='function', params=[None, 'fairdi', 'coe'])
def with_publish_to_coe_repo(monkeypatch, request):
mode = request.param
diff --git a/tests/test_api.py b/tests/test_api.py
index 5932f88ea150957560b3cb88ff0a26da53e6b557..489b1749198d0cefc565c3917285233d919ad4b2 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -447,6 +447,20 @@ class TestUploads:
self.assert_published(client, admin_user_auth, upload['upload_id'], proc_infra, metadata)
self.assert_published(client, admin_user_auth, upload['upload_id'], proc_infra, metadata, publish_with_metadata=False)
+ def test_post_re_process(self, client, published, test_user_auth, monkeypatch):
+ monkeypatch.setattr('nomad.config.version', 're_process_test_version')
+ monkeypatch.setattr('nomad.config.commit', 're_process_test_commit')
+
+ upload_id = published.upload_id
+ rv = client.post(
+ '/uploads/%s' % upload_id,
+ headers=test_user_auth,
+ data=json.dumps(dict(operation='re-process')),
+ content_type='application/json')
+
+ assert rv.status_code == 200
+ assert self.block_until_completed(client, upload_id, test_user_auth) is not None
+
# TODO validate metadata (or all input models in API for that matter)
# def test_post_bad_metadata(self, client, proc_infra, test_user_auth, postgres):
# rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)