Commit 7bacf470 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Initial refactor/restructure of nomad's cli.

parent a5a48ca9
......@@ -46,9 +46,9 @@ nomad.api
---------
.. automodule:: nomad.api
nomad.client
nomad.cli
------------
.. automodule:: nomad.client
.. automodule:: nomad.cli
nomad.utils
-----------
......
......@@ -13,12 +13,16 @@
# limitations under the License.
"""
Swagger/bravado based python client library for the API and various usefull shell commands.
Command line interface (CLI) for nomad. Provides a group/sub-command structure, think git,
that offers various functionality to the command line user.
Use it from the command line with ``nomad --help`` or ``python -m nomad.cli --help``to learn
more.
"""
from nomad.utils import POPO
from . import upload, run
from . import dev, admin, client, parse
from .cli import cli
......
......@@ -17,5 +17,4 @@ from nomad.utils import POPO
from .cli import cli
if __name__ == '__main__':
print('#######################')
cli(obj=POPO()) # pylint: disable=E1120,E1123
......@@ -12,7 +12,5 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from .main import cli
if __name__ == '__main__':
cli() # pylint: disable=E1120
from . import admin, upload, run
import click
from nomad.cli.cli import cli
@cli.group(help='''The nomad admin commands to do nasty stuff directly on the databases.
Remember: With great power comes great responsibility!''')
@click.pass_context
def admin(ctx):
pass
......@@ -13,9 +13,7 @@
# limitations under the License.
import click
import logging
import os
import sys
import shutil
from tabulate import tabulate
from elasticsearch_dsl import A
......@@ -23,47 +21,10 @@ from elasticsearch_dsl import A
from nomad import config as nomad_config, infrastructure, processing
from nomad.search import Search
@click.group(help='''The nomad admin command to do nasty stuff directly on the databases.
Remember: With great power comes great responsibility!''')
@click.option('-v', '--verbose', help='sets log level to info', is_flag=True)
@click.option('--debug', help='sets log level to debug', is_flag=True)
@click.option('--config', help='the config file to use')
@click.pass_context
def cli(ctx, verbose: bool, debug: bool, config: str):
if config is not None:
nomad_config.load_config(config_file=config)
if debug:
nomad_config.console_log_level = logging.DEBUG
elif verbose:
nomad_config.console_log_level = logging.INFO
else:
nomad_config.console_log_level = logging.WARNING
nomad_config.service = os.environ.get('NOMAD_SERVICE', 'admin')
infrastructure.setup_logging()
@cli.command(help='Runs tests and linting. Useful before commit code.')
@click.option('--skip-tests', help='Do not test, just do code checks.', is_flag=True)
def qa(skip_tests: bool):
os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
ret_code = 0
if not skip_tests:
click.echo('Run tests ...')
ret_code += os.system('python -m pytest -svx tests')
click.echo('Run code style checks ...')
ret_code += os.system('python -m pycodestyle --ignore=E501,E701 nomad tests')
click.echo('Run linter ...')
ret_code += os.system('python -m pylint --load-plugins=pylint_mongoengine nomad tests')
click.echo('Run static type checks ...')
ret_code += os.system('python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests')
sys.exit(ret_code)
from .admin import admin
@cli.command(help='Checks consistency of files and es vs mongo and deletes orphan entries.')
@admin.command(help='Checks consistency of files and es vs mongo and deletes orphan entries.')
@click.option('--dry', is_flag=True, help='Do not delete anything, just check.')
@click.option('--skip-calcs', is_flag=True, help='Skip cleaning calcs with missing uploads.')
@click.option('--skip-fs', is_flag=True, help='Skip cleaning the filesystem.')
......
......@@ -17,10 +17,10 @@ import asyncio
from concurrent.futures import ProcessPoolExecutor
from nomad import config
from .cli import cli
from .admin import admin
@cli.group(help='Run a nomad service locally (outside docker).')
@admin.group(help='Run a nomad service locally (outside docker).')
def run():
pass
......
......@@ -18,10 +18,10 @@ from mongoengine import Q
from pymongo import UpdateOne
from nomad import processing as proc, config, infrastructure, utils, search, files, coe_repo
from .cli import cli
from .admin import admin
@cli.group(help='Upload related commands')
@admin.group(help='Upload related commands')
@click.option('--user', help='Select uploads of user with given id', type=str)
@click.option('--staging', help='Select only uploads in staging', is_flag=True)
@click.option('--processing', help='Select only processing uploads', is_flag=True)
......
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import click
import logging
import os
from nomad import config as nomad_config, infrastructure
@click.group(help='''This is the entry point to nomad\'s command line interface CLI.
It uses a sub-command structure similar to the git command.''')
@click.option('-v', '--verbose', help='sets log level to info', is_flag=True)
@click.option('--debug', help='sets log level to debug', is_flag=True)
@click.option('--config', help='the config file to use')
@click.pass_context
def cli(ctx, verbose: bool, debug: bool, config: str):
if config is not None:
nomad_config.load_config(config_file=config)
if debug:
nomad_config.console_log_level = logging.DEBUG
elif verbose:
nomad_config.console_log_level = logging.INFO
else:
nomad_config.console_log_level = logging.WARNING
nomad_config.service = os.environ.get('NOMAD_SERVICE', 'admin')
infrastructure.setup_logging()
......@@ -12,14 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Swagger/bravado based python client library for the API and various usefull shell commands.
"""
from . import local, migration, upload, integrationtests, parse
from .main import cli, create_client
from . import local, migration, upload, integrationtests
from .client import create_client
from .upload import stream_upload_with_client
def run_cli():
cli() # pylint: disable=E1120
......@@ -12,17 +12,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
import requests
import click
import logging
from bravado.requests_client import RequestsClient
from bravado.client import SwaggerClient
from urllib.parse import urlparse
from nomad import config as nomad_config
from nomad import utils, infrastructure
from nomad import utils
from nomad.cli.cli import cli
def create_client():
......@@ -66,28 +65,12 @@ def handle_common_errors(func):
return wrapper
@click.group()
@cli.group(help='Commands that use the nomad API to do useful things')
@click.option('-n', '--url', default=nomad_config.client.url, help='The URL where nomad is running, default is "%s".' % nomad_config.client.url)
@click.option('-u', '--user', default=None, help='the user name to login, default is "%s" login.' % nomad_config.client.user)
@click.option('-w', '--password', default=nomad_config.client.password, help='the password used to login.')
@click.option('-v', '--verbose', help='sets log level to info', is_flag=True)
@click.option('--no-ssl-verify', help='disables SSL verificaton when talking to nomad.', is_flag=True)
@click.option('--debug', help='sets log level to debug', is_flag=True)
@click.option('--config', help='the config file to use')
def cli(url: str, verbose: bool, debug: bool, user: str, password: str, config: str, no_ssl_verify: bool):
if config is not None:
nomad_config.load_config(config_file=config)
if debug:
nomad_config.console_log_level = logging.DEBUG
elif verbose:
nomad_config.console_log_level = logging.INFO
else:
nomad_config.console_log_level = logging.WARNING
nomad_config.service = os.environ.get('NOMAD_SERVICE', 'client')
infrastructure.setup_logging()
def client(url: str, user: str, password: str, no_ssl_verify: bool):
logger = utils.get_logger(__name__)
logger.info('Used nomad is %s' % url)
......@@ -106,7 +89,6 @@ def cli(url: str, verbose: bool, debug: bool, user: str, password: str, config:
return __create_client(ssl_verify=not no_ssl_verify)
@cli.command(help='Attempts to reset the nomad.')
@client.command(help='Attempts to reset the nomad.')
def reset():
from .main import create_client
create_client().admin.exec_reset_command().response()
......@@ -19,15 +19,15 @@ as a final integration test.
import time
from .main import cli
from .client import client
example_file = 'tests/data/proc/examples_vasp.zip'
@cli.command(help='Runs a few example operations as a test.')
@client.command(help='Runs a few example operations as a test.')
def integrationtests():
from .main import create_client
from .client import create_client
client = create_client()
print('upload with multiple code data')
......
......@@ -26,9 +26,9 @@ from nomad import config, utils
from nomad.files import ArchiveBasedStagingUploadFiles
from nomad.datamodel import CalcWithMetadata
from nomad.parsing import LocalBackend
from nomad.client.parse import parse, normalize, normalize_all
from nomad.cli.parse import parse, normalize, normalize_all
from .main import cli
from .client import client
class CalcProcReproduction:
......@@ -58,7 +58,7 @@ class CalcProcReproduction:
self.mainfile = mainfile
self.parser = None
from .main import create_client
from nomad.cli.client import create_client
client = create_client()
if self.mainfile is None:
try:
......@@ -144,7 +144,7 @@ class CalcProcReproduction:
return normalize_all(parser_backend=parser_backend, logger=self.logger)
@cli.command(help='Run processing locally.')
@client.command(help='Run processing locally.')
@click.argument('CALC_ID', nargs=1, required=True, type=str)
@click.option('--override', is_flag=True, default=False, help='Override existing local calculation data.')
@click.option('--show-backend', is_flag=True, default=False, help='Print the backend data.')
......
......@@ -26,7 +26,7 @@ import json
from nomad import config, infrastructure
from nomad.migration import NomadCOEMigration, SourceCalc, Package, missing_calcs_data
from .main import cli
from .client import client
def _Migration(**kwargs) -> NomadCOEMigration:
......@@ -37,7 +37,7 @@ def _setup():
pass
@cli.group(help='Migrate data from NOMAD CoE to nomad@FAIRDI')
@client.group(help='Migrate data from NOMAD CoE to nomad@FAIRDI')
@click.option('-h', '--host', default=config.migration_source_db.host, help='The migration repository source db host, default is "%s".' % config.migration_source_db.host)
@click.option('-p', '--port', default=config.migration_source_db.port, help='The migration repository source db port, default is %d.' % config.migration_source_db.port)
@click.option('-u', '--user', default=config.migration_source_db.user, help='The migration repository source db user, default is %s.' % config.migration_source_db.user)
......
......@@ -22,7 +22,7 @@ import requests
from nomad import utils, config
from nomad.processing import FAILURE, SUCCESS
from .main import cli, create_client
from .client import client, create_client
def stream_upload_with_client(client, stream, name=None):
......@@ -93,7 +93,7 @@ def upload_file(file_path: str, name: str = None, offline: bool = False, publish
return upload.upload_id
@cli.command(
@client.command(
help='Upload files to nomad. The given path can be a single file or a directory. '
'All .zip files in a directory will be uploaded.')
@click.argument('PATH', nargs=-1, required=True, type=click.Path(exists=True))
......
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
import os
import os.path
import click
from nomad.cli.cli import cli
@cli.group(help='Commands related to the nomad source code.')
def dev():
pass
@dev.command(help='Runs tests and linting of the nomad source code. Useful before committing code.')
@click.option('--skip-tests', help='Do no tests, just do code checks.', is_flag=True)
def qa(skip_tests: bool):
os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
ret_code = 0
if not skip_tests:
click.echo('Run tests ...')
ret_code += os.system('python -m pytest -svx tests')
click.echo('Run code style checks ...')
ret_code += os.system('python -m pycodestyle --ignore=E501,E701 nomad tests')
click.echo('Run linter ...')
ret_code += os.system('python -m pylint --load-plugins=pylint_mongoengine nomad tests')
click.echo('Run static type checks ...')
ret_code += os.system('python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests')
sys.exit(ret_code)
......@@ -9,7 +9,7 @@ from nomad.parsing import LocalBackend, parser_dict, match_parser
from nomad.normalizing import normalizers
from nomad.datamodel import CalcWithMetadata
from .main import cli
from .cli import cli
def parse(
......
......@@ -16,7 +16,7 @@
This module provides function to establish connections to the database, searchengine, etc.
infrastructure services. Usually everything is setup at once with :func:`setup`. This
is run once for each *api* and *worker* process. Individual functions for partial setups
exist to facilitate testing, :py:mod:`nomad.migration`, aspects of :py:mod:`nomad.client`, etc.
exist to facilitate testing, :py:mod:`nomad.migration`, aspects of :py:mod:`nomad.cli`, etc.
"""
import os.path
......
......@@ -1030,7 +1030,7 @@ class NomadCOEMigration:
@property
def client(self):
if self._client is None:
from nomad.client import create_client
from nomad.cli.client import create_client
self._client = create_client()
return self._client
......
......@@ -18,6 +18,5 @@ setup(
install_requires=reqs,
entry_points='''
[console_scripts]
nomad=nomad.client:run_cli
admin=nomad.admin:run_cli
nomad=nomad.cli:run_cli
''')
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment