Skip to content

GitLab

Commit a5a48ca9 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Fixed issue with new yaml version. Fixed import issue with CLIs.

parent a4d26e2a
Pipeline #52342 passed with stages
in 15 minutes and 20 seconds
Hide whitespace changes
Inline Side-by-side
nomad/admin/__init__.py
View file @ a5a48ca9
......@@ -19,8 +19,8 @@ Swagger/bravado based python client library for the API and various usefull shel
from nomad.utils import POPO
from . import upload, run
from .__main__ import cli as cli_main
from .cli import cli
def cli():
cli_main(obj=POPO()) # pylint: disable=E1120,E1123
def run_cli():
cli(obj=POPO()) # pylint: disable=E1120,E1123
nomad/admin/__main__.py
View file @ a5a48ca9
......@@ -12,137 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import click
import logging
import os
import sys
import shutil
from tabulate import tabulate
from elasticsearch_dsl import A
from nomad import config as nomad_config, infrastructure, processing
from nomad.search import Search
@click.group(help='''The nomad admin command to do nasty stuff directly on the databases.
Remember: With great power comes great responsibility!''')
@click.option('-v', '--verbose', help='sets log level to info', is_flag=True)
@click.option('--debug', help='sets log level to debug', is_flag=True)
@click.option('--config', help='the config file to use')
@click.pass_context
def cli(ctx, verbose: bool, debug: bool, config: str):
if config is not None:
nomad_config.load_config(config_file=config)
if debug:
nomad_config.console_log_level = logging.DEBUG
elif verbose:
nomad_config.console_log_level = logging.INFO
else:
nomad_config.console_log_level = logging.WARNING
nomad_config.service = os.environ.get('NOMAD_SERVICE', 'admin')
infrastructure.setup_logging()
@cli.command(help='Runs tests and linting. Useful before commit code.')
@click.option('--skip-tests', help='Do not test, just do code checks.', is_flag=True)
def qa(skip_tests: bool):
os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
ret_code = 0
if not skip_tests:
click.echo('Run tests ...')
ret_code += os.system('python -m pytest -svx tests')
click.echo('Run code style checks ...')
ret_code += os.system('python -m pycodestyle --ignore=E501,E701 nomad tests')
click.echo('Run linter ...')
ret_code += os.system('python -m pylint --load-plugins=pylint_mongoengine nomad tests')
click.echo('Run static type checks ...')
ret_code += os.system('python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests')
sys.exit(ret_code)
@cli.command(help='Checks consistency of files and es vs mongo and deletes orphan entries.')
@click.option('--dry', is_flag=True, help='Do not delete anything, just check.')
@click.option('--skip-calcs', is_flag=True, help='Skip cleaning calcs with missing uploads.')
@click.option('--skip-fs', is_flag=True, help='Skip cleaning the filesystem.')
@click.option('--skip-es', is_flag=True, help='Skip cleaning the es index.')
def clean(dry, skip_calcs, skip_fs, skip_es):
infrastructure.setup_logging()
mongo_client = infrastructure.setup_mongo()
infrastructure.setup_elastic()
if not skip_calcs:
uploads_for_calcs = mongo_client[nomad_config.mongo.db_name]['calc'].distinct('upload_id')
uploads = {}
for upload in mongo_client[nomad_config.mongo.db_name]['upload'].distinct('_id'):
uploads[upload] = True
missing_uploads = []
for upload_for_calc in uploads_for_calcs:
if upload_for_calc not in uploads:
missing_uploads.append(upload_for_calc)
if not dry and len(missing_uploads) > 0:
input('Will delete calcs (mongo + es) for %d missing uploads. Press any key to continue ...' % len(missing_uploads))
for upload in missing_uploads:
mongo_client[nomad_config.mongo.db_name]['calc'].remove(dict(upload_id=upload))
Search(index=nomad_config.elastic.index_name).query('term', upload_id=upload).delete()
else:
print('Found %s uploads that have calcs in mongo, but there is no upload entry.' % len(missing_uploads))
print('List first 10:')
for upload in missing_uploads[:10]:
print(upload)
if not skip_fs:
upload_dirs = []
for bucket in [nomad_config.fs.public, nomad_config.fs.staging]:
for prefix in os.listdir(bucket):
for upload in os.listdir(os.path.join(bucket, prefix)):
upload_dirs.append((upload, os.path.join(bucket, prefix, upload)))
to_delete = list(
path for upload, path in upload_dirs
if processing.Upload.objects(upload_id=upload).first() is None)
if not dry and len(to_delete) > 0:
input('Will delete %d upload directories. Press any key to continue ...' % len(to_delete))
for path in to_delete:
shutil.rmtree(path)
else:
print('Found %d upload directories with no upload in mongo.' % len(to_delete))
print('List first 10:')
for path in to_delete[:10]:
print(path)
if not skip_es:
search = Search(index=nomad_config.elastic.index_name)
search.aggs.bucket('uploads', A('terms', field='upload_id', size=12000))
response = search.execute()
to_delete = list(
(bucket.key, bucket.doc_count)
for bucket in response.aggregations.uploads.buckets
if processing.Upload.objects(upload_id=bucket.key).first() is None)
calcs = 0
for _, upload_calcs in to_delete:
calcs += upload_calcs
if not dry and len(to_delete) > 0:
input(
'Will delete %d calcs in %d uploads from ES. Press any key to continue ...' %
(calcs, len(to_delete)))
for upload, _ in to_delete:
Search(index=nomad_config.elastic.index_name).query('term', upload_id=upload).delete()
else:
print('Found %d calcs in %d uploads from ES with no upload in mongo.' % (calcs, len(to_delete)))
print('List first 10:')
tabulate(to_delete, headers=['id', '#calcs'])
from nomad.utils import POPO
from .cli import cli
if __name__ == '__main__':
cli(obj={}) # pylint: disable=E1120,E1123
print('#######################')
cli(obj=POPO()) # pylint: disable=E1120,E1123
nomad/admin/cli.py 0 → 100644
View file @ a5a48ca9
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import click
import logging
import os
import sys
import shutil
from tabulate import tabulate
from elasticsearch_dsl import A
from nomad import config as nomad_config, infrastructure, processing
from nomad.search import Search
@click.group(help='''The nomad admin command to do nasty stuff directly on the databases.
Remember: With great power comes great responsibility!''')
@click.option('-v', '--verbose', help='sets log level to info', is_flag=True)
@click.option('--debug', help='sets log level to debug', is_flag=True)
@click.option('--config', help='the config file to use')
@click.pass_context
def cli(ctx, verbose: bool, debug: bool, config: str):
if config is not None:
nomad_config.load_config(config_file=config)
if debug:
nomad_config.console_log_level = logging.DEBUG
elif verbose:
nomad_config.console_log_level = logging.INFO
else:
nomad_config.console_log_level = logging.WARNING
nomad_config.service = os.environ.get('NOMAD_SERVICE', 'admin')
infrastructure.setup_logging()
@cli.command(help='Runs tests and linting. Useful before commit code.')
@click.option('--skip-tests', help='Do not test, just do code checks.', is_flag=True)
def qa(skip_tests: bool):
os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
ret_code = 0
if not skip_tests:
click.echo('Run tests ...')
ret_code += os.system('python -m pytest -svx tests')
click.echo('Run code style checks ...')
ret_code += os.system('python -m pycodestyle --ignore=E501,E701 nomad tests')
click.echo('Run linter ...')
ret_code += os.system('python -m pylint --load-plugins=pylint_mongoengine nomad tests')
click.echo('Run static type checks ...')
ret_code += os.system('python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests')
sys.exit(ret_code)
@cli.command(help='Checks consistency of files and es vs mongo and deletes orphan entries.')
@click.option('--dry', is_flag=True, help='Do not delete anything, just check.')
@click.option('--skip-calcs', is_flag=True, help='Skip cleaning calcs with missing uploads.')
@click.option('--skip-fs', is_flag=True, help='Skip cleaning the filesystem.')
@click.option('--skip-es', is_flag=True, help='Skip cleaning the es index.')
def clean(dry, skip_calcs, skip_fs, skip_es):
infrastructure.setup_logging()
mongo_client = infrastructure.setup_mongo()
infrastructure.setup_elastic()
if not skip_calcs:
uploads_for_calcs = mongo_client[nomad_config.mongo.db_name]['calc'].distinct('upload_id')
uploads = {}
for upload in mongo_client[nomad_config.mongo.db_name]['upload'].distinct('_id'):
uploads[upload] = True
missing_uploads = []
for upload_for_calc in uploads_for_calcs:
if upload_for_calc not in uploads:
missing_uploads.append(upload_for_calc)
if not dry and len(missing_uploads) > 0:
input('Will delete calcs (mongo + es) for %d missing uploads. Press any key to continue ...' % len(missing_uploads))
for upload in missing_uploads:
mongo_client[nomad_config.mongo.db_name]['calc'].remove(dict(upload_id=upload))
Search(index=nomad_config.elastic.index_name).query('term', upload_id=upload).delete()
else:
print('Found %s uploads that have calcs in mongo, but there is no upload entry.' % len(missing_uploads))
print('List first 10:')
for upload in missing_uploads[:10]:
print(upload)
if not skip_fs:
upload_dirs = []
for bucket in [nomad_config.fs.public, nomad_config.fs.staging]:
for prefix in os.listdir(bucket):
for upload in os.listdir(os.path.join(bucket, prefix)):
upload_dirs.append((upload, os.path.join(bucket, prefix, upload)))
to_delete = list(
path for upload, path in upload_dirs
if processing.Upload.objects(upload_id=upload).first() is None)
if not dry and len(to_delete) > 0:
input('Will delete %d upload directories. Press any key to continue ...' % len(to_delete))
for path in to_delete:
shutil.rmtree(path)
else:
print('Found %d upload directories with no upload in mongo.' % len(to_delete))
print('List first 10:')
for path in to_delete[:10]:
print(path)
if not skip_es:
search = Search(index=nomad_config.elastic.index_name)
search.aggs.bucket('uploads', A('terms', field='upload_id', size=12000))
response = search.execute()
to_delete = list(
(bucket.key, bucket.doc_count)
for bucket in response.aggregations.uploads.buckets
if processing.Upload.objects(upload_id=bucket.key).first() is None)
calcs = 0
for _, upload_calcs in to_delete:
calcs += upload_calcs
if not dry and len(to_delete) > 0:
input(
'Will delete %d calcs in %d uploads from ES. Press any key to continue ...' %
(calcs, len(to_delete)))
for upload, _ in to_delete:
Search(index=nomad_config.elastic.index_name).query('term', upload_id=upload).delete()
else:
print('Found %d calcs in %d uploads from ES with no upload in mongo.' % (calcs, len(to_delete)))
print('List first 10:')
tabulate(to_delete, headers=['id', '#calcs'])
nomad/admin/run.py
View file @ a5a48ca9
......@@ -17,7 +17,7 @@ import asyncio
from concurrent.futures import ProcessPoolExecutor
from nomad import config
from nomad.admin.__main__ import cli
from .cli import cli
@cli.group(help='Run a nomad service locally (outside docker).')
......
nomad/admin/upload.py
View file @ a5a48ca9
......@@ -18,7 +18,7 @@ from mongoengine import Q
from pymongo import UpdateOne
from nomad import processing as proc, config, infrastructure, utils, search, files, coe_repo
from .__main__ import cli
from .cli import cli
@cli.group(help='Upload related commands')
......@@ -42,7 +42,7 @@ def upload(ctx, user: str, staging: bool, processing: bool, outdated: bool):
if outdated:
uploads = proc.Calc._get_collection().distinct(
'upload_id',
{'metadata.nomad_version': { '$ne': config.version}})
{'metadata.nomad_version': {'$ne': config.version}})
query &= Q(upload_id__in=uploads)
ctx.obj.query = query
......@@ -156,13 +156,13 @@ def re_process(ctx, uploads):
logger = utils.get_logger(__name__)
print('%d uploads selected, re-processing ...' % uploads.count())
def re_process_upload(upload: str):
def re_process_upload(upload):
logger.info('re-processing started', upload_id=upload.upload_id)
upload.re_process_upload()
upload.block_until_complete(interval=.1)
logger.info('re-processing complete', upload_id=upload_id)
logger.info('re-processing complete', upload_id=upload.upload_id)
count = 0
for upload in uploads:
......
nomad/api/upload.py
View file @ a5a48ca9
......@@ -445,7 +445,7 @@ class UploadResource(Resource):
@login_really_required
def post(self, upload_id):
"""
Execute an upload operation. Available operations: ``publish``
Execute an upload operation. Available operations are ``publish`` and ``re-process``
Publish accepts further meta data that allows to provide coauthors, comments,
external references, etc. See the model for details. The fields that start with
......@@ -453,6 +453,10 @@ class UploadResource(Resource):
Publish changes the visibility of the upload. Clients can specify the visibility
via meta data.
Re-process will re-process the upload and produce updated repository metadata and
archive. Only published uploads that are not processing at the moment are allowed.
Only for uploads where calculations have been processed with an older nomad version.
"""
try:
upload = Upload.get(upload_id)
......@@ -489,8 +493,21 @@ class UploadResource(Resource):
abort(400, message='The upload is still/already processed')
return upload, 200
elif operation == 're-process':
if upload.tasks_running or not upload.published:
abort(400, message='Can only non processing, re-process published uploads')
if len(metadata) > 0:
abort(400, message='You can not provide metadata for re-processing')
if len(upload.outdated_calcs) == 0:
abort(400, message='You can only re-process uploads with at least one outdated calculation')
upload.re_process_upload()
return upload, 200
abort(400, message='Unsuported operation %s.' % operation)
abort(400, message='Unsupported operation %s.' % operation)
upload_command_model = api.model('UploadCommand', {
......
nomad/client/__init__.py
View file @ a5a48ca9
......@@ -17,5 +17,9 @@ Swagger/bravado based python client library for the API and various usefull shel
"""
from . import local, migration, upload, integrationtests, parse
from .__main__ import cli, create_client
from .main import cli, create_client
from .upload import stream_upload_with_client
def run_cli():
cli() # pylint: disable=E1120
nomad/client/__main__.py
View file @ a5a48ca9
......@@ -12,106 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
import requests
import click
import logging
from bravado.requests_client import RequestsClient
from bravado.client import SwaggerClient
from urllib.parse import urlparse
import nomad.client
from nomad import config as nomad_config
from nomad import utils, infrastructure
def create_client():
return _create_client()
def _create_client(*args, **kwargs):
return __create_client(*args, **kwargs)
def __create_client(user: str = nomad_config.client.user, password: str = nomad_config.client.password, ssl_verify: bool = True):
""" A factory method to create the client. """
host = urlparse(nomad_config.client.url).netloc.split(':')[0]
if not ssl_verify:
import warnings
warnings.filterwarnings("ignore")
http_client = RequestsClient(ssl_verify=ssl_verify)
if user is not None:
http_client.set_basic_auth(host, user, password)
client = SwaggerClient.from_url(
'%s/swagger.json' % nomad_config.client.url,
http_client=http_client)
utils.get_logger(__name__).info('created bravado client', user=user)
return client
def handle_common_errors(func):
def wrapper(*args, **kwargs):
try:
func(*args, **kwargs)
except requests.exceptions.ConnectionError:
click.echo(
'\nCould not connect to nomad at %s. '
'Check connection and url.' % nomad_config.client.url)
sys.exit(0)
return wrapper
@click.group()
@click.option('-n', '--url', default=nomad_config.client.url, help='The URL where nomad is running, default is "%s".' % nomad_config.client.url)
@click.option('-u', '--user', default=None, help='the user name to login, default is "%s" login.' % nomad_config.client.user)
@click.option('-w', '--password', default=nomad_config.client.password, help='the password used to login.')
@click.option('-v', '--verbose', help='sets log level to info', is_flag=True)
@click.option('--no-ssl-verify', help='disables SSL verificaton when talking to nomad.', is_flag=True)
@click.option('--debug', help='sets log level to debug', is_flag=True)
@click.option('--config', help='the config file to use')
def cli(url: str, verbose: bool, debug: bool, user: str, password: str, config: str, no_ssl_verify: bool):
if config is not None:
nomad_config.load_config(config_file=config)
if debug:
nomad_config.console_log_level = logging.DEBUG
elif verbose:
nomad_config.console_log_level = logging.INFO
else:
nomad_config.console_log_level = logging.WARNING
nomad_config.service = os.environ.get('NOMAD_SERVICE', 'client')
infrastructure.setup_logging()
logger = utils.get_logger(__name__)
logger.info('Used nomad is %s' % url)
logger.info('Used user is %s' % user)
nomad_config.client.url = url
global _create_client
def _create_client(*args, **kwargs): # pylint: disable=W0612
if user is not None:
logger.info('create client', user=user)
return __create_client(user=user, password=password, ssl_verify=not no_ssl_verify)
else:
logger.info('create anonymous client')
return __create_client(ssl_verify=not no_ssl_verify)
@cli.command(help='Attempts to reset the nomad.')
def reset():
from .__main__ import create_client
create_client().admin.exec_reset_command().response()
from .main import cli
if __name__ == '__main__':
nomad.client.cli() # pylint: disable=E1120
cli() # pylint: disable=E1120
nomad/client/integrationtests.py
View file @ a5a48ca9
......@@ -19,7 +19,7 @@ as a final integration test.
import time
from .__main__ import cli
from .main import cli
example_file = 'tests/data/proc/examples_vasp.zip'
......@@ -27,7 +27,7 @@ example_file = 'tests/data/proc/examples_vasp.zip'
@cli.command(help='Runs a few example operations as a test.')
def integrationtests():
from .__main__ import create_client
from .main import create_client
client = create_client()
print('upload with multiple code data')
......
nomad/client/local.py
View file @ a5a48ca9
......@@ -28,7 +28,7 @@ from nomad.datamodel import CalcWithMetadata
from nomad.parsing import LocalBackend
from nomad.client.parse import parse, normalize, normalize_all
from .__main__ import cli
from .main import cli
class CalcProcReproduction:
......@@ -58,7 +58,7 @@ class CalcProcReproduction:
self.mainfile = mainfile
self.parser = None
from .__main__ import create_client
from .main import create_client
client = create_client()
if self.mainfile is None:
try:
......
nomad/client/main.py 0 → 100644
View file @ a5a48ca9
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
import requests
import click
import logging
from bravado.requests_client import RequestsClient
from bravado.client import SwaggerClient
from urllib.parse import urlparse
from nomad import config as nomad_config
from nomad import utils, infrastructure
def create_client():
return _create_client()
def _create_client(*args, **kwargs):
return __create_client(*args, **kwargs)
def __create_client(user: str = nomad_config.client.user, password: str = nomad_config.client.password, ssl_verify: bool = True):
""" A factory method to create the client. """
host = urlparse(nomad_config.client.url).netloc.split(':')[0]
if not ssl_verify:
import warnings
warnings.filterwarnings("ignore")
http_client = RequestsClient(ssl_verify=ssl_verify)
if user is not None:
http_client.set_basic_auth(host, user, password)
client = SwaggerClient.from_url(
'%s/swagger.json' % nomad_config.client.url,
http_client=http_client)
utils.get_logger(__name__).info('created bravado client', user=user)
return client