Initial refactor/restructure of nomad's cli.

docs/reference.rst

@@ -46,9 +46,9 @@ nomad.api
 ---------
 .. automodule:: nomad.api
 
-nomad.client
+nomad.cli
 ------------
-.. automodule:: nomad.client
+.. automodule:: nomad.cli
 
 nomad.utils
 -----------

nomad/admin/__init__.py → nomad/cli/__init__.py

@@ -13,12 +13,16 @@
 # limitations under the License.
 
 """
-Swagger/bravado based python client library for the API and various usefull shell commands.
+Command line interface (CLI) for nomad. Provides a group/sub-command structure, think git,
+that offers various functionality to the command line user.
+
+Use it from the command line with ``nomad --help`` or ``python -m nomad.cli --help``to learn
+more.
 """
 
 from nomad.utils import POPO
 
-from . import upload, run
+from . import dev, admin, client, parse
 from .cli import cli
 
 

nomad/admin/__main__.py → nomad/cli/__main__.py

@@ -17,5 +17,4 @@ from nomad.utils import POPO
 from .cli import cli
 
 if __name__ == '__main__':
-    print('#######################')
     cli(obj=POPO())  # pylint: disable=E1120,E1123

nomad/client/__main__.py → nomad/cli/admin/__init__.py

@@ -12,7 +12,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .main import cli
 
-if __name__ == '__main__':
-    cli()  # pylint: disable=E1120
+from . import admin, upload, run

nomad/cli/admin/admin.py

+import click
+
+from nomad.cli.cli import cli
+
+
+@cli.group(help='''The nomad admin commands to do nasty stuff directly on the databases.
+                     Remember: With great power comes great responsibility!''')
+@click.pass_context
+def admin(ctx):
+    pass

nomad/admin/cli.py → nomad/cli/admin/clean.py

@@ -13,9 +13,7 @@
 # limitations under the License.
 
 import click
-import logging
 import os
-import sys
 import shutil
 from tabulate import tabulate
 from elasticsearch_dsl import A
@@ -23,47 +21,10 @@ from elasticsearch_dsl import A
 from nomad import config as nomad_config, infrastructure, processing
 from nomad.search import Search
 
-
-@click.group(help='''The nomad admin command to do nasty stuff directly on the databases.
-                     Remember: With great power comes great responsibility!''')
-@click.option('-v', '--verbose', help='sets log level to info', is_flag=True)
-@click.option('--debug', help='sets log level to debug', is_flag=True)
-@click.option('--config', help='the config file to use')
-@click.pass_context
-def cli(ctx, verbose: bool, debug: bool, config: str):
-    if config is not None:
-        nomad_config.load_config(config_file=config)
-
-    if debug:
-        nomad_config.console_log_level = logging.DEBUG
-    elif verbose:
-        nomad_config.console_log_level = logging.INFO
-    else:
-        nomad_config.console_log_level = logging.WARNING
-
-    nomad_config.service = os.environ.get('NOMAD_SERVICE', 'admin')
-    infrastructure.setup_logging()
-
-
-@cli.command(help='Runs tests and linting. Useful before commit code.')
-@click.option('--skip-tests', help='Do not test, just do code checks.', is_flag=True)
-def qa(skip_tests: bool):
-    os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
-    ret_code = 0
-    if not skip_tests:
-        click.echo('Run tests ...')
-        ret_code += os.system('python -m pytest -svx tests')
-    click.echo('Run code style checks ...')
-    ret_code += os.system('python -m pycodestyle --ignore=E501,E701 nomad tests')
-    click.echo('Run linter ...')
-    ret_code += os.system('python -m pylint --load-plugins=pylint_mongoengine nomad tests')
-    click.echo('Run static type checks ...')
-    ret_code += os.system('python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests')
-
-    sys.exit(ret_code)
+from .admin import admin
 
 
-@cli.command(help='Checks consistency of files and es vs mongo and deletes orphan entries.')
+@admin.command(help='Checks consistency of files and es vs mongo and deletes orphan entries.')
 @click.option('--dry', is_flag=True, help='Do not delete anything, just check.')
 @click.option('--skip-calcs', is_flag=True, help='Skip cleaning calcs with missing uploads.')
 @click.option('--skip-fs', is_flag=True, help='Skip cleaning the filesystem.')

nomad/admin/run.py → nomad/cli/admin/run.py

@@ -17,10 +17,10 @@ import asyncio
 from concurrent.futures import ProcessPoolExecutor
 
 from nomad import config
-from .cli import cli
+from .admin import admin
 
 
-@cli.group(help='Run a nomad service locally (outside docker).')
+@admin.group(help='Run a nomad service locally (outside docker).')
 def run():
     pass
 

nomad/admin/upload.py → nomad/cli/admin/upload.py

@@ -18,10 +18,10 @@ from mongoengine import Q
 from pymongo import UpdateOne
 
 from nomad import processing as proc, config, infrastructure, utils, search, files, coe_repo
-from .cli import cli
+from .admin import admin
 
 
-@cli.group(help='Upload related commands')
+@admin.group(help='Upload related commands')
 @click.option('--user', help='Select uploads of user with given id', type=str)
 @click.option('--staging', help='Select only uploads in staging', is_flag=True)
 @click.option('--processing', help='Select only processing uploads', is_flag=True)

nomad/cli/cli.py

+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+import logging
+import os
+
+from nomad import config as nomad_config, infrastructure
+
+
+@click.group(help='''This is the entry point to nomad\'s command line interface CLI.
+                     It uses a sub-command structure similar to the git command.''')
+@click.option('-v', '--verbose', help='sets log level to info', is_flag=True)
+@click.option('--debug', help='sets log level to debug', is_flag=True)
+@click.option('--config', help='the config file to use')
+@click.pass_context
+def cli(ctx, verbose: bool, debug: bool, config: str):
+    if config is not None:
+        nomad_config.load_config(config_file=config)
+
+    if debug:
+        nomad_config.console_log_level = logging.DEBUG
+    elif verbose:
+        nomad_config.console_log_level = logging.INFO
+    else:
+        nomad_config.console_log_level = logging.WARNING
+
+    nomad_config.service = os.environ.get('NOMAD_SERVICE', 'admin')
+    infrastructure.setup_logging()

nomad/client/__init__.py → nomad/cli/client/__init__.py

@@ -12,14 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""
-Swagger/bravado based python client library for the API and various usefull shell commands.
-"""
 
-from . import local, migration, upload, integrationtests, parse
-from .main import cli, create_client
+from . import local, migration, upload, integrationtests
+from .client import create_client
 from .upload import stream_upload_with_client
-
-
-def run_cli():
-    cli()  # pylint: disable=E1120

nomad/client/main.py → nomad/cli/client/client.py

@@ -12,17 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
 import sys
 import requests
 import click
-import logging
 from bravado.requests_client import RequestsClient
 from bravado.client import SwaggerClient
 from urllib.parse import urlparse
 
 from nomad import config as nomad_config
-from nomad import utils, infrastructure
+from nomad import utils
+from nomad.cli.cli import cli
 
 
 def create_client():
@@ -66,28 +65,12 @@ def handle_common_errors(func):
     return wrapper
 
 
-@click.group()
+@cli.group(help='Commands that use the nomad API to do useful things')
 @click.option('-n', '--url', default=nomad_config.client.url, help='The URL where nomad is running, default is "%s".' % nomad_config.client.url)
 @click.option('-u', '--user', default=None, help='the user name to login, default is "%s" login.' % nomad_config.client.user)
 @click.option('-w', '--password', default=nomad_config.client.password, help='the password used to login.')
-@click.option('-v', '--verbose', help='sets log level to info', is_flag=True)
 @click.option('--no-ssl-verify', help='disables SSL verificaton when talking to nomad.', is_flag=True)
-@click.option('--debug', help='sets log level to debug', is_flag=True)
-@click.option('--config', help='the config file to use')
-def cli(url: str, verbose: bool, debug: bool, user: str, password: str, config: str, no_ssl_verify: bool):
-    if config is not None:
-        nomad_config.load_config(config_file=config)
-
-    if debug:
-        nomad_config.console_log_level = logging.DEBUG
-    elif verbose:
-        nomad_config.console_log_level = logging.INFO
-    else:
-        nomad_config.console_log_level = logging.WARNING
-
-    nomad_config.service = os.environ.get('NOMAD_SERVICE', 'client')
-    infrastructure.setup_logging()
-
+def client(url: str, user: str, password: str, no_ssl_verify: bool):
     logger = utils.get_logger(__name__)
 
     logger.info('Used nomad is %s' % url)
@@ -106,7 +89,6 @@ def cli(url: str, verbose: bool, debug: bool, user: str, password: str, config:
             return __create_client(ssl_verify=not no_ssl_verify)
 
 
-@cli.command(help='Attempts to reset the nomad.')
+@client.command(help='Attempts to reset the nomad.')
 def reset():
-    from .main import create_client
     create_client().admin.exec_reset_command().response()

nomad/client/integrationtests.py → nomad/cli/client/integrationtests.py

@@ -19,15 +19,15 @@ as a final integration test.
 
 import time
 
-from .main import cli
+from .client import client
 
 
 example_file = 'tests/data/proc/examples_vasp.zip'
 
 
-@cli.command(help='Runs a few example operations as a test.')
+@client.command(help='Runs a few example operations as a test.')
 def integrationtests():
-    from .main import create_client
+    from .client import create_client
     client = create_client()
 
     print('upload with multiple code data')

nomad/client/local.py → nomad/cli/client/local.py

@@ -26,9 +26,9 @@ from nomad import config, utils
 from nomad.files import ArchiveBasedStagingUploadFiles
 from nomad.datamodel import CalcWithMetadata
 from nomad.parsing import LocalBackend
-from nomad.client.parse import parse, normalize, normalize_all
+from nomad.cli.parse import parse, normalize, normalize_all
 
-from .main import cli
+from .client import client
 
 
 class CalcProcReproduction:
@@ -58,7 +58,7 @@ class CalcProcReproduction:
         self.mainfile = mainfile
         self.parser = None
 
-        from .main import create_client
+        from nomad.cli.client import create_client
         client = create_client()
         if self.mainfile is None:
             try:
@@ -144,7 +144,7 @@ class CalcProcReproduction:
         return normalize_all(parser_backend=parser_backend, logger=self.logger)
 
 
-@cli.command(help='Run processing locally.')
+@client.command(help='Run processing locally.')
 @click.argument('CALC_ID', nargs=1, required=True, type=str)
 @click.option('--override', is_flag=True, default=False, help='Override existing local calculation data.')
 @click.option('--show-backend', is_flag=True, default=False, help='Print the backend data.')

nomad/client/migration.py → nomad/cli/client/migration.py

@@ -26,7 +26,7 @@ import json
 from nomad import config, infrastructure
 from nomad.migration import NomadCOEMigration, SourceCalc, Package, missing_calcs_data
 
-from .main import cli
+from .client import client
 
 
 def _Migration(**kwargs) -> NomadCOEMigration:
@@ -37,7 +37,7 @@ def _setup():
     pass
 
 
-@cli.group(help='Migrate data from NOMAD CoE to nomad@FAIRDI')
+@client.group(help='Migrate data from NOMAD CoE to nomad@FAIRDI')
 @click.option('-h', '--host', default=config.migration_source_db.host, help='The migration repository source db host, default is "%s".' % config.migration_source_db.host)
 @click.option('-p', '--port', default=config.migration_source_db.port, help='The migration repository source db port, default is %d.' % config.migration_source_db.port)
 @click.option('-u', '--user', default=config.migration_source_db.user, help='The migration repository source db user, default is %s.' % config.migration_source_db.user)

nomad/client/upload.py → nomad/cli/client/upload.py

@@ -22,7 +22,7 @@ import requests
 from nomad import utils, config
 from nomad.processing import FAILURE, SUCCESS
 
-from .main import cli, create_client
+from .client import client, create_client
 
 
 def stream_upload_with_client(client, stream, name=None):
@@ -93,7 +93,7 @@ def upload_file(file_path: str, name: str = None, offline: bool = False, publish
     return upload.upload_id
 
 
-@cli.command(
+@client.command(
     help='Upload files to nomad. The given path can be a single file or a directory. '
     'All .zip files in a directory will be uploaded.')
 @click.argument('PATH', nargs=-1, required=True, type=click.Path(exists=True))

nomad/cli/dev.py

+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import os
+import os.path
+import click
+
+from nomad.cli.cli import cli
+
+
+@cli.group(help='Commands related to the nomad source code.')
+def dev():
+    pass
+
+
+@dev.command(help='Runs tests and linting of the nomad source code. Useful before committing code.')
+@click.option('--skip-tests', help='Do no tests, just do code checks.', is_flag=True)
+def qa(skip_tests: bool):
+    os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
+    ret_code = 0
+    if not skip_tests:
+        click.echo('Run tests ...')
+        ret_code += os.system('python -m pytest -svx tests')
+    click.echo('Run code style checks ...')
+    ret_code += os.system('python -m pycodestyle --ignore=E501,E701 nomad tests')
+    click.echo('Run linter ...')
+    ret_code += os.system('python -m pylint --load-plugins=pylint_mongoengine nomad tests')
+    click.echo('Run static type checks ...')
+    ret_code += os.system('python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests')
+
+    sys.exit(ret_code)

nomad/client/parse.py → nomad/cli/parse.py

@@ -9,7 +9,7 @@ from nomad.parsing import LocalBackend, parser_dict, match_parser
 from nomad.normalizing import normalizers
 from nomad.datamodel import CalcWithMetadata
 
-from .main import cli
+from .cli import cli
 
 
 def parse(

nomad/infrastructure.py

@@ -16,7 +16,7 @@
 This module provides function to establish connections to the database, searchengine, etc.
 infrastructure services. Usually everything is setup at once with :func:`setup`. This
 is run once for each *api* and *worker* process. Individual functions for partial setups
-exist to facilitate testing, :py:mod:`nomad.migration`, aspects of :py:mod:`nomad.client`, etc.
+exist to facilitate testing, :py:mod:`nomad.migration`, aspects of :py:mod:`nomad.cli`, etc.
 """
 
 import os.path

nomad/migration.py

@@ -1030,7 +1030,7 @@ class NomadCOEMigration:
     @property
     def client(self):
         if self._client is None:
-            from nomad.client import create_client
+            from nomad.cli.client import create_client
             self._client = create_client()
 
         return self._client

setup.py

@@ -18,6 +18,5 @@ setup(
     install_requires=reqs,
     entry_points='''
         [console_scripts]
-        nomad=nomad.client:run_cli
-        admin=nomad.admin:run_cli
+        nomad=nomad.cli:run_cli
     ''')