Defered the exection of cli group functions to increase cli loadup speed.

nomad/cli/__init__.py

@@ -33,13 +33,15 @@ lazy_import.lazy_module('nomad.utils')
 lazy_import.lazy_module('nomad.parsing')
 lazy_import.lazy_module('nomad.normalizing')
 lazy_import.lazy_module('nomad.datamodel')
+lazy_import.lazy_module('nomad.search')
+lazy_import.lazy_module('nomad.metainfo')
+lazy_import.lazy_module('nomad.processing')
+lazy_import.lazy_module('nomad.client')
 lazy_import.lazy_module('nomadcore')
 
-from nomad.utils import POPO  # noqa
-
-from . import dev, admin, client, parse  # noqa
+from . import dev, parse, admin, client  # noqa
 from .cli import cli  # noqa
 
 
 def run_cli():
-    cli(obj=POPO())  # pylint: disable=E1120,E1123
+    cli()  # pylint: disable=E1120,E1123

nomad/cli/__main__.py

@@ -12,9 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from nomad.utils import POPO
-
 from .cli import cli
 
 if __name__ == '__main__':
-    cli(obj=POPO())  # pylint: disable=E1120,E1123
+    cli()  # pylint: disable=E1120,E1123

nomad/cli/cli.py

@@ -19,7 +19,78 @@ import os
 from nomad import config as nomad_config, infrastructure
 
 
-@click.group(help='''This is the entry point to nomad\'s command line interface CLI.
+class LazyCommand(click.Command):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.orig_callback = self.callback
+        self.callback = self.lazy_callback
+
+    def lazy_callback(self, *args, **kwargs):
+        for group_callback, group_args, group_kwargs in self.ctx.obj.group_invocations:
+            group_callback(*group_args, **group_kwargs)
+        return self.orig_callback(*args, **kwargs)
+
+    def invoke(self, ctx):
+        self.ctx = ctx
+        return super().invoke(ctx)
+
+
+class LazyGroup(click.Group):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.orig_callback = self.callback
+
+        self.callback = self.lazy_callback
+        self.ctx = None
+
+    def lazy_callback(self, *args, **kwargs):
+        self.ctx.obj.group_invocations.append((self.orig_callback, args, kwargs))
+        return None
+
+    def command(self, *args, **kwargs):
+        kwargs.setdefault('cls', LazyCommand)
+        return super().command(*args, **kwargs)
+
+    def group(self, *args, **kwargs):
+        kwargs.setdefault('cls', LazyGroup)
+        return super().group(*args, **kwargs)
+
+    def invoke(self, ctx):
+        if ctx.obj is None:
+            ctx.obj = POPO()
+            ctx.obj.group_invocations = []
+
+        self.ctx = ctx
+        return super().invoke(ctx)
+
+
+class POPO(dict):
+    '''
+    A dict subclass that uses attributes as key/value pairs.
+    '''
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def __getattr__(self, name):
+        if name in self:
+            return self[name]
+        else:
+            raise AttributeError("No such attribute: " + name)
+
+    def __setattr__(self, name, value):
+        self[name] = value
+
+    def __delattr__(self, name):
+        if name in self:
+            del self[name]
+        else:
+            raise AttributeError("No such attribute: " + name)
+
+
+@click.group(
+    cls=LazyGroup,
+    help='''This is the entry point to nomad\'s command line interface CLI.
                      It uses a sub-command structure similar to the git command.''')
 @click.option('-v', '--verbose', help='sets log level to info', is_flag=True)
 @click.option('--debug', help='sets log level to debug', is_flag=True)

nomad/cli/client/__init__.py

@@ -36,6 +36,7 @@ lazy_import.lazy_module('time')
 lazy_import.lazy_module('matplotlib.scale')
 lazy_import.lazy_module('matplotlib.pyplot')
 lazy_import.lazy_module('matplotlib.ticker')
+lazy_import.lazy_module('matplotlib.transforms')
 lazy_import.lazy_module('numpy')
 lazy_import.lazy_module('nomad.config')
 lazy_import.lazy_module('nomad.utils')
@@ -46,6 +47,7 @@ lazy_import.lazy_module('nomad.datamodel')
 lazy_import.lazy_module('nomad.parsing')
 lazy_import.lazy_module('nomad.infrastructure')
 lazy_import.lazy_module('nomad.doi')
+lazy_import.lazy_module('nomad.client')
 
 from . import local, upload, integrationtests, mirror, statistics, update_database  # noqa
 from .client import create_client  # noqa

nomad/cli/client/client.py

@@ -18,39 +18,12 @@ import click
 from bravado import requests_client as bravado_requests_client
 from bravado import client as bravado_client
 from urllib import parse as urllib_parse
-from keycloak import KeycloakOpenID
-import time
 
 from nomad import config as nomad_config
 from nomad import utils
-from nomad.cli.cli import cli
-
+from nomad import client as nomad_client
 
-class KeycloakAuthenticator(bravado_requests_client.Authenticator):
-    def __init__(self, host, user, password, **kwargs):
-        super().__init__(host=host)
-        self.user = user
-        self.password = password
-        self.token = None
-        self.__oidc = KeycloakOpenID(**kwargs)
-
-    def apply(self, request=None):
-        if self.token is None:
-            self.token = self.__oidc.token(username=self.user, password=self.password)
-            self.token['time'] = time.time()
-        elif self.token['expires_in'] < int(time.time()) - self.token['time'] + 10:
-            try:
-                self.token = self.__oidc.refresh_token(self.token['refresh_token'])
-                self.token['time'] = time.time()
-            except Exception:
-                self.token = self.__oidc.token(username=self.user, password=self.password)
-                self.token['time'] = time.time()
-
-        if request:
-            request.headers.setdefault('Authorization', 'Bearer %s' % self.token['access_token'])
-            return request
-        else:
-            return dict(Authorization='Bearer %s' % self.token['access_token'])
+from nomad.cli.cli import cli
 
 
 def create_client():
@@ -81,7 +54,7 @@ def __create_client(
     if user is not None:
         host = urllib_parse.urlparse(nomad_config.client.url).netloc.split(':')[0]
         if use_token:
-            http_client.authenticator = KeycloakAuthenticator(
+            http_client.authenticator = nomad_client.KeycloakAuthenticator(
                 host=host,
                 user=user,
                 password=password,

nomad/cli/client/mirror.py

@@ -29,13 +29,13 @@ from nomad.cli.admin import uploads as admin_uploads
 from .client import client
 
 
-__mongo_properties = set(d.name for d in datamodel.MongoMetadata.m_def.definitions)
-
 __in_test = False
 ''' Will be monkeypatched by tests to alter behavior for testing. '''
 
-_Dataset = datamodel.Dataset.m_def.a_mongo.mongo_cls
-__logger = utils.get_logger(__name__)
+
+# will be initialized in mirror command func
+_Dataset = None
+__logger = None
 
 
 def fix_time(data, keys):
@@ -85,6 +85,7 @@ def transform_reference(reference):
 
 def v0Dot7(upload_data):
     ''' Inplace transforms v0.7.x upload data into v0.8.x upload data. '''
+    __mongo_properties = set(d.name for d in datamodel.MongoMetadata.m_def.definitions)
     for calc in upload_data['calcs']:
         calc_metadata = calc['metadata']
         if 'pid' in calc_metadata:
@@ -197,6 +198,12 @@ def mirror(
         source_mapping: str, target_mapping: str, migration: str, staging: bool,
         skip_es: bool, replace: bool):
 
+    # init global vars
+    global _Dataset
+    global __logger
+    _Dataset = datamodel.Dataset.m_def.a_mongo.mongo_cls
+    __logger = utils.get_logger(__name__)
+
     if staging and not (move or link):
         print('--with-staging requires either --move or --link')
         sys.exit(1)

nomad/cli/client/statistics.py

@@ -16,8 +16,8 @@
 A command that generates various statistics.
 '''
 
-from matplotlib import scale as mscale
-from matplotlib import transforms as mtransforms
+import matplotlib.scale as mscale
+import matplotlib.transforms as mtransforms
 import matplotlib.pyplot as plt
 import matplotlib.ticker as ticker
 import numpy as np

nomad/client.py

@@ -41,13 +41,39 @@ import requests
 from urllib.parse import urlparse
 
 from nomad import config
-from nomad.cli.client.client import KeycloakAuthenticator
 from nomad.datamodel import EntryArchive
 
 # TODO this import is necessary to load all metainfo defintions that the parsers are using
 from nomad import parsing  # pylint: disable=unused-import
 
 
+class KeycloakAuthenticator(bravado_requests_client.Authenticator):
+    def __init__(self, host, user, password, **kwargs):
+        super().__init__(host=host)
+        self.user = user
+        self.password = password
+        self.token = None
+        self.__oidc = KeycloakOpenID(**kwargs)
+
+    def apply(self, request=None):
+        if self.token is None:
+            self.token = self.__oidc.token(username=self.user, password=self.password)
+            self.token['time'] = time.time()
+        elif self.token['expires_in'] < int(time.time()) - self.token['time'] + 10:
+            try:
+                self.token = self.__oidc.refresh_token(self.token['refresh_token'])
+                self.token['time'] = time.time()
+            except Exception:
+                self.token = self.__oidc.token(username=self.user, password=self.password)
+                self.token['time'] = time.time()
+
+        if request:
+            request.headers.setdefault('Authorization', 'Bearer %s' % self.token['access_token'])
+            return request
+        else:
+            return dict(Authorization='Bearer %s' % self.token['access_token'])
+
+
 class ArchiveQuery(Sequence):
     def __init__(
             self,

nomad/utils.py

@@ -467,29 +467,6 @@ def chunks(list, n):
         yield list[i:i + n]
 
 
-class POPO(dict):
-    '''
-    A dict subclass that uses attributes as key/value pairs.
-    '''
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-    def __getattr__(self, name):
-        if name in self:
-            return self[name]
-        else:
-            raise AttributeError("No such attribute: " + name)
-
-    def __setattr__(self, name, value):
-        self[name] = value
-
-    def __delattr__(self, name):
-        if name in self:
-            del self[name]
-        else:
-            raise AttributeError("No such attribute: " + name)
-
-
 class SleepTimeBackoff:
     '''
     Provides increasingly larger sleeps. Useful when

requirements.txt

@@ -80,3 +80,4 @@ essential_generators
 jinja2
 bs4
 html5lib
+fastentrypoints

setup.py

@@ -6,6 +6,8 @@ try:  # for pip >= 10
 except ImportError:  # for pip <= 9.0.3
     from pip.req import parse_requirements
 
+import fastentrypoints
+
 # parse_requirements() returns generator of pip.req.InstallRequirement objects
 install_reqs = parse_requirements('requirements.txt', session='hack')
 # reqs is a list of requirement

tests/test_cli.py

@@ -19,8 +19,9 @@ import json
 import datetime
 import zipfile
 
-from nomad import utils, search, processing as proc, files
+from nomad import search, processing as proc, files
 from nomad.cli import cli
+from nomad.cli.cli import POPO
 from nomad.processing import Upload, Calc
 
 from tests.app.test_app import BlueprintClient
@@ -32,7 +33,7 @@ from tests.app.test_app import BlueprintClient
 class TestAdmin:
     def test_reset(self, reset_infra):
         result = click.testing.CliRunner().invoke(
-            cli, ['admin', 'reset', '--i-am-really-sure'], catch_exceptions=False, obj=utils.POPO())
+            cli, ['admin', 'reset', '--i-am-really-sure'], catch_exceptions=False)
         assert result.exit_code == 0
 
         # allow other test to re-establish a connection
@@ -40,12 +41,12 @@ class TestAdmin:
 
     def test_reset_not_sure(self):
         result = click.testing.CliRunner().invoke(
-            cli, ['admin', 'reset'], catch_exceptions=False, obj=utils.POPO())
+            cli, ['admin', 'reset'], catch_exceptions=False)
         assert result.exit_code == 1
 
     # def test_remove(self, reset_infra):
     #     result = click.testing.CliRunner().invoke(
-    #         cli, ['admin', 'reset', '--remove', '--i-am-really-sure'], catch_exceptions=False, obj=utils.POPO())
+    #         cli, ['admin', 'reset', '--remove', '--i-am-really-sure'], catch_exceptions=False)
     #     assert result.exit_code == 0
     #     # allow other test to re-establish a connection
     #     mongoengine.disconnect_all()
@@ -59,7 +60,7 @@ class TestAdmin:
         assert search.SearchRequest().search_parameter('upload_id', upload_id).execute()['total'] > 0
 
         result = click.testing.CliRunner().invoke(
-            cli, ['admin', 'clean', '--force', '--skip-es'], catch_exceptions=False, obj=utils.POPO())
+            cli, ['admin', 'clean', '--force', '--skip-es'], catch_exceptions=False)
 
         assert result.exit_code == 0
         assert not published.upload_files.exists()
@@ -85,7 +86,7 @@ class TestAdmin:
 
         result = click.testing.CliRunner().invoke(
             cli, ['admin', 'lift-embargo'] + (['--dry'] if dry else []),
-            catch_exceptions=False, obj=utils.POPO())
+            catch_exceptions=False)
 
         assert result.exit_code == 0
         assert not Calc.objects(upload_id=upload_id).first().metadata['with_embargo'] == lifted
@@ -103,7 +104,7 @@ class TestAdmin:
         assert search.SearchRequest().search_parameter('comment', 'specific').execute()['total'] == 0
 
         result = click.testing.CliRunner().invoke(
-            cli, ['admin', 'index', '--threads', '2'], catch_exceptions=False, obj=utils.POPO())
+            cli, ['admin', 'index', '--threads', '2'], catch_exceptions=False)
         assert result.exit_code == 0
         assert 'index' in result.stdout
 
@@ -114,7 +115,7 @@ class TestAdmin:
         calc = Calc.objects(upload_id=upload_id).first()
 
         result = click.testing.CliRunner().invoke(
-            cli, ['admin', 'entries', 'rm', calc.calc_id], catch_exceptions=False, obj=utils.POPO())
+            cli, ['admin', 'entries', 'rm', calc.calc_id], catch_exceptions=False)
 
         assert result.exit_code == 0
         assert 'deleting' in result.stdout
@@ -135,7 +136,7 @@ class TestAdminUploads:
             codes_args.append('--code')
             codes_args.append(code)
         result = click.testing.CliRunner().invoke(
-            cli, ['admin', 'uploads'] + codes_args + ['ls'], catch_exceptions=False, obj=utils.POPO())
+            cli, ['admin', 'uploads'] + codes_args + ['ls'], catch_exceptions=False)
 
         assert result.exit_code == 0
         assert '%d uploads selected' % count in result.stdout
@@ -146,7 +147,7 @@ class TestAdminUploads:
         query = dict(upload_id=upload_id)
         result = click.testing.CliRunner().invoke(
             cli, ['admin', 'uploads', '--query-mongo', 'ls', json.dumps(query)],
-            catch_exceptions=False, obj=utils.POPO())
+            catch_exceptions=False)
 
         assert result.exit_code == 0
         assert '1 uploads selected' in result.stdout
@@ -155,7 +156,7 @@ class TestAdminUploads:
         upload_id = published.upload_id
 
         result = click.testing.CliRunner().invoke(
-            cli, ['admin', 'uploads', 'ls', upload_id], catch_exceptions=False, obj=utils.POPO())
+            cli, ['admin', 'uploads', 'ls', upload_id], catch_exceptions=False)
 
         assert result.exit_code == 0
         assert '1 uploads selected' in result.stdout
@@ -164,7 +165,7 @@ class TestAdminUploads:
         upload_id = published.upload_id
 
         result = click.testing.CliRunner().invoke(
-            cli, ['admin', 'uploads', 'ls', '{"match":{"upload_id":"%s"}}' % upload_id], catch_exceptions=False, obj=utils.POPO())
+            cli, ['admin', 'uploads', 'ls', '{"match":{"upload_id":"%s"}}' % upload_id], catch_exceptions=False)
 
         assert result.exit_code == 0
         assert '1 uploads selected' in result.stdout
@@ -173,7 +174,7 @@ class TestAdminUploads:
         upload_id = published.upload_id
 
         result = click.testing.CliRunner().invoke(
-            cli, ['admin', 'uploads', 'rm', upload_id], catch_exceptions=False, obj=utils.POPO())
+            cli, ['admin', 'uploads', 'rm', upload_id], catch_exceptions=False)
 
         assert result.exit_code == 0
         assert 'deleting' in result.stdout
@@ -191,7 +192,7 @@ class TestAdminUploads:
                         f.write(json.dumps(dict(archive='test')).encode())
 
         result = click.testing.CliRunner().invoke(
-            cli, ['admin', 'uploads', 'msgpack', upload_id], catch_exceptions=False, obj=utils.POPO())
+            cli, ['admin', 'uploads', 'msgpack', upload_id], catch_exceptions=False)
 
         assert result.exit_code == 0
         assert 'wrote msgpack archive' in result.stdout
@@ -207,7 +208,7 @@ class TestAdminUploads:
         assert search.SearchRequest().search_parameters(comment='specific').execute()['total'] == 0
 
         result = click.testing.CliRunner().invoke(
-            cli, ['admin', 'uploads', 'index', upload_id], catch_exceptions=False, obj=utils.POPO())
+            cli, ['admin', 'uploads', 'index', upload_id], catch_exceptions=False)
         assert result.exit_code == 0
         assert 'index' in result.stdout
 
@@ -220,7 +221,7 @@ class TestAdminUploads:
         assert calc.metadata['nomad_version'] != 'test_version'
 
         result = click.testing.CliRunner().invoke(
-            cli, ['admin', 'uploads', 're-process', '--parallel', '2', upload_id], catch_exceptions=False, obj=utils.POPO())
+            cli, ['admin', 'uploads', 're-process', '--parallel', '2', upload_id], catch_exceptions=False)
 
         assert result.exit_code == 0
         assert 're-processing' in result.stdout
@@ -235,7 +236,7 @@ class TestAdminUploads:
         calc.save()
 
         result = click.testing.CliRunner().invoke(
-            cli, ['admin', 'uploads', 're-pack', '--parallel', '2', upload_id], catch_exceptions=False, obj=utils.POPO())
+            cli, ['admin', 'uploads', 're-pack', '--parallel', '2', upload_id], catch_exceptions=False)
 
         assert result.exit_code == 0
         assert 're-pack' in result.stdout
@@ -254,7 +255,7 @@ class TestAdminUploads:
         assert calc.metadata['uploader'] == other_test_user.user_id
 
         result = click.testing.CliRunner().invoke(
-            cli, ['admin', 'uploads', 'chown', test_user.username, upload_id], catch_exceptions=False, obj=utils.POPO())
+            cli, ['admin', 'uploads', 'chown', test_user.username, upload_id], catch_exceptions=False)
 
         assert result.exit_code == 0
         assert 'changing' in result.stdout
@@ -269,7 +270,7 @@ class TestAdminUploads:
         upload_id = non_empty_processed.upload_id
 
         result = click.testing.CliRunner().invoke(
-            cli, ['admin', 'uploads', 'reset', '--with-calcs', upload_id], catch_exceptions=False, obj=utils.POPO())
+            cli, ['admin', 'uploads', 'reset', '--with-calcs', upload_id], catch_exceptions=False)
 
         assert result.exit_code == 0
         assert 'reset' in result.stdout
@@ -287,7 +288,7 @@ class TestClient:
         result = click.testing.CliRunner().invoke(
             cli,
             ['client', 'upload', '--offline', '--name', 'test_upload', non_empty_example_upload],
-            catch_exceptions=False, obj=utils.POPO())
+            catch_exceptions=False)
 
         assert result.exit_code == 0
         assert '1/0/1' in result.output
@@ -298,13 +299,13 @@ class TestClient:
             assert stream
             rv = client.get(url[url.index('/api/raw'):], headers=headers)
             assert rv.status_code == 200
-            return utils.POPO(iter_content=lambda *args, **kwargs: [bytes(rv.data)])
+            return POPO(iter_content=lambda *args, **kwargs: [bytes(rv.data)])
 
         monkeypatch.setattr('requests.get', requests_get)
         result = click.testing.CliRunner().invoke(
             cli,
             ['client', 'local', '%s/%s' % (published.upload_id, list(published.calcs)[0].calc_id)],
-            catch_exceptions=False, obj=utils.POPO())
+            catch_exceptions=False)
 
         assert result.exit_code == 0
 
@@ -312,7 +313,7 @@ class TestClient:
         monkeypatch.setattr('nomad.cli.client.mirror.__in_test', True)
 
         result = click.testing.CliRunner().invoke(
-            cli, ['client', 'mirror', '--dry'], catch_exceptions=False, obj=utils.POPO())
+            cli, ['client', 'mirror', '--dry'], catch_exceptions=False)
 
         assert result.exit_code == 0
         assert published.upload_id in result.output
@@ -328,13 +329,13 @@ class TestClient:
 
         if move:
             result = click.testing.CliRunner().invoke(
-                cli, ['client', 'mirror', '--move'], catch_exceptions=False, obj=utils.POPO())
+                cli, ['client', 'mirror', '--move'], catch_exceptions=False)
         elif link:
             result = click.testing.CliRunner().invoke(
-                cli, ['client', 'mirror', '--link'], catch_exceptions=False, obj=utils.POPO())
+                cli, ['client', 'mirror', '--link'], catch_exceptions=False)
         else:
             result = click.testing.CliRunner().invoke(
-                cli, ['client', 'mirror', '--source-mapping', '.volumes/test_fs:.volumes/test_fs'], catch_exceptions=False, obj=utils.POPO())
+                cli, ['client', 'mirror', '--source-mapping', '.volumes/test_fs:.volumes/test_fs'], catch_exceptions=False)
 
         assert result.exit_code == 0
         assert published.upload_id in result.output
@@ -363,7 +364,7 @@ class TestClient:
         monkeypatch.setattr('nomad.cli.client.mirror.__in_test', True)
 
         result = click.testing.CliRunner().invoke(
-            cli, ['client', 'mirror', '--staging', '--link'], catch_exceptions=False, obj=utils.POPO())
+            cli, ['client', 'mirror', '--staging', '--link'], catch_exceptions=False)
 
         assert result.exit_code == 0
         assert non_empty_processed.upload_id in result.output
@@ -386,7 +387,7 @@ class TestClient:
         monkeypatch.setattr('nomad.cli.client.mirror.__in_test', True)
 
         result = click.testing.CliRunner().invoke(
-            cli, ['client', 'mirror', '--files-only'], catch_exceptions=False, obj=utils.POPO())
+            cli, ['client', 'mirror', '--files-only'], catch_exceptions=False)
 
         assert result.exit_code == 0, result.output
         assert published.upload_id in result.output
@@ -415,7 +416,7 @@ class TestClient:
         monkeypatch.setattr('nomad.cli.client.mirror.__in_test', True)
 
         result = click.testing.CliRunner().invoke(
-            cli, ['client', 'mirror'], catch_exceptions=False, obj=utils.POPO())
+            cli, ['client', 'mirror'], catch_exceptions=False)
 
         assert result.exit_code == 0, result.output
         assert published_wo_user_metadata.upload_id in result.output
@@ -426,7 +427,7 @@ class TestClient:
     def test_statistics(self, client, proc_infra, admin_user_bravado_client):
 
         result = click.testing.CliRunner().invoke(
-            cli, ['client', 'statistics-table'], catch_exceptions=True, obj=utils.POPO())
+            cli, ['client', 'statistics-table'], catch_exceptions=True)
 
         assert result.exit_code == 0, result.output
         assert 'Calculations, e.g. total energies' in result.output