Commit e9b97ed4 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Defered the exection of cli group functions to increase cli loadup speed.

parent a5e94cab
Pipeline #72266 failed with stages
in 17 minutes and 54 seconds
......@@ -33,13 +33,15 @@ lazy_import.lazy_module('nomad.utils')
lazy_import.lazy_module('nomad.parsing')
lazy_import.lazy_module('nomad.normalizing')
lazy_import.lazy_module('nomad.datamodel')
lazy_import.lazy_module('nomad.search')
lazy_import.lazy_module('nomad.metainfo')
lazy_import.lazy_module('nomad.processing')
lazy_import.lazy_module('nomad.client')
lazy_import.lazy_module('nomadcore')
from nomad.utils import POPO # noqa
from . import dev, admin, client, parse # noqa
from . import dev, parse, admin, client # noqa
from .cli import cli # noqa
def run_cli():
cli(obj=POPO()) # pylint: disable=E1120,E1123
cli() # pylint: disable=E1120,E1123
......@@ -12,9 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from nomad.utils import POPO
from .cli import cli
if __name__ == '__main__':
cli(obj=POPO()) # pylint: disable=E1120,E1123
cli() # pylint: disable=E1120,E1123
......@@ -19,7 +19,78 @@ import os
from nomad import config as nomad_config, infrastructure
@click.group(help='''This is the entry point to nomad\'s command line interface CLI.
class LazyCommand(click.Command):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.orig_callback = self.callback
self.callback = self.lazy_callback
def lazy_callback(self, *args, **kwargs):
for group_callback, group_args, group_kwargs in self.ctx.obj.group_invocations:
group_callback(*group_args, **group_kwargs)
return self.orig_callback(*args, **kwargs)
def invoke(self, ctx):
self.ctx = ctx
return super().invoke(ctx)
class LazyGroup(click.Group):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.orig_callback = self.callback
self.callback = self.lazy_callback
self.ctx = None
def lazy_callback(self, *args, **kwargs):
self.ctx.obj.group_invocations.append((self.orig_callback, args, kwargs))
return None
def command(self, *args, **kwargs):
kwargs.setdefault('cls', LazyCommand)
return super().command(*args, **kwargs)
def group(self, *args, **kwargs):
kwargs.setdefault('cls', LazyGroup)
return super().group(*args, **kwargs)
def invoke(self, ctx):
if ctx.obj is None:
ctx.obj = POPO()
ctx.obj.group_invocations = []
self.ctx = ctx
return super().invoke(ctx)
class POPO(dict):
'''
A dict subclass that uses attributes as key/value pairs.
'''
def __init__(self, **kwargs):
super().__init__(**kwargs)
def __getattr__(self, name):
if name in self:
return self[name]
else:
raise AttributeError("No such attribute: " + name)
def __setattr__(self, name, value):
self[name] = value
def __delattr__(self, name):
if name in self:
del self[name]
else:
raise AttributeError("No such attribute: " + name)
@click.group(
cls=LazyGroup,
help='''This is the entry point to nomad\'s command line interface CLI.
It uses a sub-command structure similar to the git command.''')
@click.option('-v', '--verbose', help='sets log level to info', is_flag=True)
@click.option('--debug', help='sets log level to debug', is_flag=True)
......
......@@ -36,6 +36,7 @@ lazy_import.lazy_module('time')
lazy_import.lazy_module('matplotlib.scale')
lazy_import.lazy_module('matplotlib.pyplot')
lazy_import.lazy_module('matplotlib.ticker')
lazy_import.lazy_module('matplotlib.transforms')
lazy_import.lazy_module('numpy')
lazy_import.lazy_module('nomad.config')
lazy_import.lazy_module('nomad.utils')
......@@ -46,6 +47,7 @@ lazy_import.lazy_module('nomad.datamodel')
lazy_import.lazy_module('nomad.parsing')
lazy_import.lazy_module('nomad.infrastructure')
lazy_import.lazy_module('nomad.doi')
lazy_import.lazy_module('nomad.client')
from . import local, upload, integrationtests, mirror, statistics, update_database # noqa
from .client import create_client # noqa
......
......@@ -18,39 +18,12 @@ import click
from bravado import requests_client as bravado_requests_client
from bravado import client as bravado_client
from urllib import parse as urllib_parse
from keycloak import KeycloakOpenID
import time
from nomad import config as nomad_config
from nomad import utils
from nomad.cli.cli import cli
from nomad import client as nomad_client
class KeycloakAuthenticator(bravado_requests_client.Authenticator):
def __init__(self, host, user, password, **kwargs):
super().__init__(host=host)
self.user = user
self.password = password
self.token = None
self.__oidc = KeycloakOpenID(**kwargs)
def apply(self, request=None):
if self.token is None:
self.token = self.__oidc.token(username=self.user, password=self.password)
self.token['time'] = time.time()
elif self.token['expires_in'] < int(time.time()) - self.token['time'] + 10:
try:
self.token = self.__oidc.refresh_token(self.token['refresh_token'])
self.token['time'] = time.time()
except Exception:
self.token = self.__oidc.token(username=self.user, password=self.password)
self.token['time'] = time.time()
if request:
request.headers.setdefault('Authorization', 'Bearer %s' % self.token['access_token'])
return request
else:
return dict(Authorization='Bearer %s' % self.token['access_token'])
from nomad.cli.cli import cli
def create_client():
......@@ -81,7 +54,7 @@ def __create_client(
if user is not None:
host = urllib_parse.urlparse(nomad_config.client.url).netloc.split(':')[0]
if use_token:
http_client.authenticator = KeycloakAuthenticator(
http_client.authenticator = nomad_client.KeycloakAuthenticator(
host=host,
user=user,
password=password,
......
......@@ -29,13 +29,13 @@ from nomad.cli.admin import uploads as admin_uploads
from .client import client
__mongo_properties = set(d.name for d in datamodel.MongoMetadata.m_def.definitions)
__in_test = False
''' Will be monkeypatched by tests to alter behavior for testing. '''
_Dataset = datamodel.Dataset.m_def.a_mongo.mongo_cls
__logger = utils.get_logger(__name__)
# will be initialized in mirror command func
_Dataset = None
__logger = None
def fix_time(data, keys):
......@@ -85,6 +85,7 @@ def transform_reference(reference):
def v0Dot7(upload_data):
''' Inplace transforms v0.7.x upload data into v0.8.x upload data. '''
__mongo_properties = set(d.name for d in datamodel.MongoMetadata.m_def.definitions)
for calc in upload_data['calcs']:
calc_metadata = calc['metadata']
if 'pid' in calc_metadata:
......@@ -197,6 +198,12 @@ def mirror(
source_mapping: str, target_mapping: str, migration: str, staging: bool,
skip_es: bool, replace: bool):
# init global vars
global _Dataset
global __logger
_Dataset = datamodel.Dataset.m_def.a_mongo.mongo_cls
__logger = utils.get_logger(__name__)
if staging and not (move or link):
print('--with-staging requires either --move or --link')
sys.exit(1)
......
......@@ -16,8 +16,8 @@
A command that generates various statistics.
'''
from matplotlib import scale as mscale
from matplotlib import transforms as mtransforms
import matplotlib.scale as mscale
import matplotlib.transforms as mtransforms
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
import numpy as np
......
......@@ -41,13 +41,39 @@ import requests
from urllib.parse import urlparse
from nomad import config
from nomad.cli.client.client import KeycloakAuthenticator
from nomad.datamodel import EntryArchive
# TODO this import is necessary to load all metainfo defintions that the parsers are using
from nomad import parsing # pylint: disable=unused-import
class KeycloakAuthenticator(bravado_requests_client.Authenticator):
def __init__(self, host, user, password, **kwargs):
super().__init__(host=host)
self.user = user
self.password = password
self.token = None
self.__oidc = KeycloakOpenID(**kwargs)
def apply(self, request=None):
if self.token is None:
self.token = self.__oidc.token(username=self.user, password=self.password)
self.token['time'] = time.time()
elif self.token['expires_in'] < int(time.time()) - self.token['time'] + 10:
try:
self.token = self.__oidc.refresh_token(self.token['refresh_token'])
self.token['time'] = time.time()
except Exception:
self.token = self.__oidc.token(username=self.user, password=self.password)
self.token['time'] = time.time()
if request:
request.headers.setdefault('Authorization', 'Bearer %s' % self.token['access_token'])
return request
else:
return dict(Authorization='Bearer %s' % self.token['access_token'])
class ArchiveQuery(Sequence):
def __init__(
self,
......
......@@ -467,29 +467,6 @@ def chunks(list, n):
yield list[i:i + n]
class POPO(dict):
'''
A dict subclass that uses attributes as key/value pairs.
'''
def __init__(self, **kwargs):
super().__init__(**kwargs)
def __getattr__(self, name):
if name in self:
return self[name]
else:
raise AttributeError("No such attribute: " + name)
def __setattr__(self, name, value):
self[name] = value
def __delattr__(self, name):
if name in self:
del self[name]
else:
raise AttributeError("No such attribute: " + name)
class SleepTimeBackoff:
'''
Provides increasingly larger sleeps. Useful when
......
......@@ -80,3 +80,4 @@ essential_generators
jinja2
bs4
html5lib
fastentrypoints
......@@ -6,6 +6,8 @@ try: # for pip >= 10
except ImportError: # for pip <= 9.0.3
from pip.req import parse_requirements
import fastentrypoints
# parse_requirements() returns generator of pip.req.InstallRequirement objects
install_reqs = parse_requirements('requirements.txt', session='hack')
# reqs is a list of requirement
......
......@@ -19,8 +19,9 @@ import json
import datetime
import zipfile
from nomad import utils, search, processing as proc, files
from nomad import search, processing as proc, files
from nomad.cli import cli
from nomad.cli.cli import POPO
from nomad.processing import Upload, Calc
from tests.app.test_app import BlueprintClient
......@@ -32,7 +33,7 @@ from tests.app.test_app import BlueprintClient
class TestAdmin:
def test_reset(self, reset_infra):
result = click.testing.CliRunner().invoke(
cli, ['admin', 'reset', '--i-am-really-sure'], catch_exceptions=False, obj=utils.POPO())
cli, ['admin', 'reset', '--i-am-really-sure'], catch_exceptions=False)
assert result.exit_code == 0
# allow other test to re-establish a connection
......@@ -40,12 +41,12 @@ class TestAdmin:
def test_reset_not_sure(self):
result = click.testing.CliRunner().invoke(
cli, ['admin', 'reset'], catch_exceptions=False, obj=utils.POPO())
cli, ['admin', 'reset'], catch_exceptions=False)
assert result.exit_code == 1
# def test_remove(self, reset_infra):
# result = click.testing.CliRunner().invoke(
# cli, ['admin', 'reset', '--remove', '--i-am-really-sure'], catch_exceptions=False, obj=utils.POPO())
# cli, ['admin', 'reset', '--remove', '--i-am-really-sure'], catch_exceptions=False)
# assert result.exit_code == 0
# # allow other test to re-establish a connection
# mongoengine.disconnect_all()
......@@ -59,7 +60,7 @@ class TestAdmin:
assert search.SearchRequest().search_parameter('upload_id', upload_id).execute()['total'] > 0
result = click.testing.CliRunner().invoke(
cli, ['admin', 'clean', '--force', '--skip-es'], catch_exceptions=False, obj=utils.POPO())
cli, ['admin', 'clean', '--force', '--skip-es'], catch_exceptions=False)
assert result.exit_code == 0
assert not published.upload_files.exists()
......@@ -85,7 +86,7 @@ class TestAdmin:
result = click.testing.CliRunner().invoke(
cli, ['admin', 'lift-embargo'] + (['--dry'] if dry else []),
catch_exceptions=False, obj=utils.POPO())
catch_exceptions=False)
assert result.exit_code == 0
assert not Calc.objects(upload_id=upload_id).first().metadata['with_embargo'] == lifted
......@@ -103,7 +104,7 @@ class TestAdmin:
assert search.SearchRequest().search_parameter('comment', 'specific').execute()['total'] == 0
result = click.testing.CliRunner().invoke(
cli, ['admin', 'index', '--threads', '2'], catch_exceptions=False, obj=utils.POPO())
cli, ['admin', 'index', '--threads', '2'], catch_exceptions=False)
assert result.exit_code == 0
assert 'index' in result.stdout
......@@ -114,7 +115,7 @@ class TestAdmin:
calc = Calc.objects(upload_id=upload_id).first()
result = click.testing.CliRunner().invoke(
cli, ['admin', 'entries', 'rm', calc.calc_id], catch_exceptions=False, obj=utils.POPO())
cli, ['admin', 'entries', 'rm', calc.calc_id], catch_exceptions=False)
assert result.exit_code == 0
assert 'deleting' in result.stdout
......@@ -135,7 +136,7 @@ class TestAdminUploads:
codes_args.append('--code')
codes_args.append(code)
result = click.testing.CliRunner().invoke(
cli, ['admin', 'uploads'] + codes_args + ['ls'], catch_exceptions=False, obj=utils.POPO())
cli, ['admin', 'uploads'] + codes_args + ['ls'], catch_exceptions=False)
assert result.exit_code == 0
assert '%d uploads selected' % count in result.stdout
......@@ -146,7 +147,7 @@ class TestAdminUploads:
query = dict(upload_id=upload_id)
result = click.testing.CliRunner().invoke(
cli, ['admin', 'uploads', '--query-mongo', 'ls', json.dumps(query)],
catch_exceptions=False, obj=utils.POPO())
catch_exceptions=False)
assert result.exit_code == 0
assert '1 uploads selected' in result.stdout
......@@ -155,7 +156,7 @@ class TestAdminUploads:
upload_id = published.upload_id
result = click.testing.CliRunner().invoke(
cli, ['admin', 'uploads', 'ls', upload_id], catch_exceptions=False, obj=utils.POPO())
cli, ['admin', 'uploads', 'ls', upload_id], catch_exceptions=False)
assert result.exit_code == 0
assert '1 uploads selected' in result.stdout
......@@ -164,7 +165,7 @@ class TestAdminUploads:
upload_id = published.upload_id
result = click.testing.CliRunner().invoke(
cli, ['admin', 'uploads', 'ls', '{"match":{"upload_id":"%s"}}' % upload_id], catch_exceptions=False, obj=utils.POPO())
cli, ['admin', 'uploads', 'ls', '{"match":{"upload_id":"%s"}}' % upload_id], catch_exceptions=False)
assert result.exit_code == 0
assert '1 uploads selected' in result.stdout
......@@ -173,7 +174,7 @@ class TestAdminUploads:
upload_id = published.upload_id
result = click.testing.CliRunner().invoke(
cli, ['admin', 'uploads', 'rm', upload_id], catch_exceptions=False, obj=utils.POPO())
cli, ['admin', 'uploads', 'rm', upload_id], catch_exceptions=False)
assert result.exit_code == 0
assert 'deleting' in result.stdout
......@@ -191,7 +192,7 @@ class TestAdminUploads:
f.write(json.dumps(dict(archive='test')).encode())
result = click.testing.CliRunner().invoke(
cli, ['admin', 'uploads', 'msgpack', upload_id], catch_exceptions=False, obj=utils.POPO())
cli, ['admin', 'uploads', 'msgpack', upload_id], catch_exceptions=False)
assert result.exit_code == 0
assert 'wrote msgpack archive' in result.stdout
......@@ -207,7 +208,7 @@ class TestAdminUploads:
assert search.SearchRequest().search_parameters(comment='specific').execute()['total'] == 0
result = click.testing.CliRunner().invoke(
cli, ['admin', 'uploads', 'index', upload_id], catch_exceptions=False, obj=utils.POPO())
cli, ['admin', 'uploads', 'index', upload_id], catch_exceptions=False)
assert result.exit_code == 0
assert 'index' in result.stdout
......@@ -220,7 +221,7 @@ class TestAdminUploads:
assert calc.metadata['nomad_version'] != 'test_version'
result = click.testing.CliRunner().invoke(
cli, ['admin', 'uploads', 're-process', '--parallel', '2', upload_id], catch_exceptions=False, obj=utils.POPO())
cli, ['admin', 'uploads', 're-process', '--parallel', '2', upload_id], catch_exceptions=False)
assert result.exit_code == 0
assert 're-processing' in result.stdout
......@@ -235,7 +236,7 @@ class TestAdminUploads:
calc.save()
result = click.testing.CliRunner().invoke(
cli, ['admin', 'uploads', 're-pack', '--parallel', '2', upload_id], catch_exceptions=False, obj=utils.POPO())
cli, ['admin', 'uploads', 're-pack', '--parallel', '2', upload_id], catch_exceptions=False)
assert result.exit_code == 0
assert 're-pack' in result.stdout
......@@ -254,7 +255,7 @@ class TestAdminUploads:
assert calc.metadata['uploader'] == other_test_user.user_id
result = click.testing.CliRunner().invoke(
cli, ['admin', 'uploads', 'chown', test_user.username, upload_id], catch_exceptions=False, obj=utils.POPO())
cli, ['admin', 'uploads', 'chown', test_user.username, upload_id], catch_exceptions=False)
assert result.exit_code == 0
assert 'changing' in result.stdout
......@@ -269,7 +270,7 @@ class TestAdminUploads:
upload_id = non_empty_processed.upload_id
result = click.testing.CliRunner().invoke(
cli, ['admin', 'uploads', 'reset', '--with-calcs', upload_id], catch_exceptions=False, obj=utils.POPO())
cli, ['admin', 'uploads', 'reset', '--with-calcs', upload_id], catch_exceptions=False)
assert result.exit_code == 0
assert 'reset' in result.stdout
......@@ -287,7 +288,7 @@ class TestClient:
result = click.testing.CliRunner().invoke(
cli,
['client', 'upload', '--offline', '--name', 'test_upload', non_empty_example_upload],
catch_exceptions=False, obj=utils.POPO())
catch_exceptions=False)
assert result.exit_code == 0
assert '1/0/1' in result.output
......@@ -298,13 +299,13 @@ class TestClient:
assert stream
rv = client.get(url[url.index('/api/raw'):], headers=headers)
assert rv.status_code == 200
return utils.POPO(iter_content=lambda *args, **kwargs: [bytes(rv.data)])
return POPO(iter_content=lambda *args, **kwargs: [bytes(rv.data)])
monkeypatch.setattr('requests.get', requests_get)
result = click.testing.CliRunner().invoke(
cli,
['client', 'local', '%s/%s' % (published.upload_id, list(published.calcs)[0].calc_id)],
catch_exceptions=False, obj=utils.POPO())
catch_exceptions=False)
assert result.exit_code == 0
......@@ -312,7 +313,7 @@ class TestClient:
monkeypatch.setattr('nomad.cli.client.mirror.__in_test', True)
result = click.testing.CliRunner().invoke(
cli, ['client', 'mirror', '--dry'], catch_exceptions=False, obj=utils.POPO())
cli, ['client', 'mirror', '--dry'], catch_exceptions=False)
assert result.exit_code == 0
assert published.upload_id in result.output
......@@ -328,13 +329,13 @@ class TestClient:
if move:
result = click.testing.CliRunner().invoke(
cli, ['client', 'mirror', '--move'], catch_exceptions=False, obj=utils.POPO())
cli, ['client', 'mirror', '--move'], catch_exceptions=False)
elif link:
result = click.testing.CliRunner().invoke(
cli, ['client', 'mirror', '--link'], catch_exceptions=False, obj=utils.POPO())
cli, ['client', 'mirror', '--link'], catch_exceptions=False)
else:
result = click.testing.CliRunner().invoke(
cli, ['client', 'mirror', '--source-mapping', '.volumes/test_fs:.volumes/test_fs'], catch_exceptions=False, obj=utils.POPO())
cli, ['client', 'mirror', '--source-mapping', '.volumes/test_fs:.volumes/test_fs'], catch_exceptions=False)
assert result.exit_code == 0
assert published.upload_id in result.output
......@@ -363,7 +364,7 @@ class TestClient:
monkeypatch.setattr('nomad.cli.client.mirror.__in_test', True)
result = click.testing.CliRunner().invoke(
cli, ['client', 'mirror', '--staging', '--link'], catch_exceptions=False, obj=utils.POPO())
cli, ['client', 'mirror', '--staging', '--link'], catch_exceptions=False)
assert result.exit_code == 0
assert non_empty_processed.upload_id in result.output
......@@ -386,7 +387,7 @@ class TestClient:
monkeypatch.setattr('nomad.cli.client.mirror.__in_test', True)
result = click.testing.CliRunner().invoke(
cli, ['client', 'mirror', '--files-only'], catch_exceptions=False, obj=utils.POPO())
cli, ['client', 'mirror', '--files-only'], catch_exceptions=False)
assert result.exit_code == 0, result.output
assert published.upload_id in result.output
......@@ -415,7 +416,7 @@ class TestClient:
monkeypatch.setattr('nomad.cli.client.mirror.__in_test', True)
result = click.testing.CliRunner().invoke(
cli, ['client', 'mirror'], catch_exceptions=False, obj=utils.POPO())
cli, ['client', 'mirror'], catch_exceptions=False)
assert result.exit_code == 0, result.output
assert published_wo_user_metadata.upload_id in result.output
......@@ -426,7 +427,7 @@ class TestClient:
def test_statistics(self, client, proc_infra, admin_user_bravado_client):
result = click.testing.CliRunner().invoke(
cli, ['client', 'statistics-table'], catch_exceptions=True, obj=utils.POPO())
cli, ['client', 'statistics-table'], catch_exceptions=True)
assert result.exit_code == 0, result.output
assert 'Calculations, e.g. total energies' in result.output
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment