Skip to content
GitLab
Commit abad0d7e authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Refactored client code. Refactored repo_db test setup/teardown. Added... 

Refactored client code. Refactored repo_db test setup/teardown. Added migration index functionality.
parent a3a01373
Pipeline #42494 passed with stages
in 20 minutes and 50 seconds
Hide whitespace changes
Inline Side-by-side
nomad/api/auth.py
View file @ abad0d7e
......@@ -69,7 +69,8 @@ def verify_password(username_or_token, password):
# try to authenticate with username/password
try:
g.user = User.verify_user_password(username_or_token, password)
except Exception:
except Exception as e:
utils.get_logger(__name__).error('could not verify password', exc_info=e)
return False
if not g.user:
......
nomad/client/__init__.py 0 → 100644
View file @ abad0d7e
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Swagger/bravado based python client library for the API and various usefull shell commands.
"""
from . import local, migration, misc, upload
from .main import cli
if __name__ == '__main__':
cli() # pylint: disable=E1120
nomad/client.py → nomad/client/local.py
View file @ abad0d7e
......@@ -12,113 +12,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Swagger/bravado based python client library for the API and various usefull shell commands.
"""
import os.path
import os
import sys
import time
import requests
import click
from typing import Union, Callable, cast
import logging
from bravado.requests_client import RequestsClient
from bravado.client import SwaggerClient
from nomad import config, utils
from nomad.files import ArchiveBasedStagingUploadFiles
from nomad.parsing import parsers, parser_dict, LocalBackend
from nomad.normalizing import normalizers
api_base = 'http://%s:%d/%s' % (config.services.api_host, config.services.api_port, config.services.api_base_path)
user = 'leonard.hofstadter@nomad-fairdi.tests.de'
pw = 'password'
def _cli_client():
return create_client()
def create_client(
host: str = config.services.api_host,
port: int = config.services.api_port,
base_path: str = config.services.api_base_path,
user: str = user, password: str = pw):
""" A factory method to create the client. """
http_client = RequestsClient()
if user is not None:
http_client.set_basic_auth(host, user, pw)
client = SwaggerClient.from_url(
'http://%s:%d%s/swagger.json' % (host, port, base_path),
http_client=http_client)
return client
def handle_common_errors(func):
def wrapper(*args, **kwargs):
try:
func(*args, **kwargs)
except requests.exceptions.ConnectionError:
click.echo(
'\nCould not connect to nomad at %s. '
'Check connection and host/port options.' % api_base)
sys.exit(0)
return wrapper
def upload_file(file_path: str, name: str = None, offline: bool = False, commit: bool = False, client=None):
"""
Upload a file to nomad.
Arguments:
file_path: path to the file, absolute or relative to call directory
name: optional name, default is the file_path's basename
offline: allows to process data without upload, requires client to be run on the server
commit: automatically commit after successful processing
Returns: The upload_id
"""
if client is None:
client = _cli_client()
if offline:
upload = client.uploads.upload(
local_path=os.path.abspath(file_path), name=name).reponse().result
click.echo('process offline: %s' % file_path)
else:
with open(file_path, 'rb') as f:
upload = client.uploads.upload(file=f, name=name).response().result
click.echo('process online: %s' % file_path)
while upload.tasks_status not in ['SUCCESS', 'FAILURE']:
upload = client.uploads.get_upload(upload_id=upload.upload_id).response().result
calcs = upload.calcs.pagination
if calcs is None:
total, successes, failures = 0, 0, 0
else:
total, successes, failures = (calcs.total, calcs.successes, calcs.failures)
ret = '\n' if upload.tasks_status in ('SUCCESS', 'FAILURE') else '\r'
print(
'status: %s; task: %s; parsing: %d/%d/%d %s' %
(upload.tasks_status, upload.current_task, successes, failures, total, ret), end='')
time.sleep(3)
if upload.tasks_status == 'FAILURE':
click.echo('There have been errors:')
for error in upload.errors:
click.echo(' %s' % error)
elif commit:
client.uploads.exec_upload_command(upload_id=upload.upload_id, operation='commit').reponse()
return upload.upload_id
from .main import cli, api_base
class CalcProcReproduction:
......@@ -131,10 +36,6 @@ class CalcProcReproduction:
the upload, archive ids to given by ELK, and reproduce and fix the error
in your development environment.
This is a class of :class:`UploadFile` the downloaded raw data will be treated as
an fake 'upload' that only contains the respective calculation data. This allows us
to locally run processing code that is very similar to the one used on the server.
Arguments:
archive_id: The archive_id of the calculation to locally process.
override: Set to true to override any existing local calculation data.
......@@ -235,71 +136,6 @@ class CalcProcReproduction:
return parser_backend
@click.group()
@click.option('-h', '--host', default=config.services.api_host, help='The host nomad runs on, default is "%s".' % config.services.api_host)
@click.option('-p', '--port', default=config.services.api_port, help='the port nomad runs with, default is %d.' % config.services.api_port)
@click.option('-u', '--user', default=None, help='the user name to login, default no login.')
@click.option('-w', '--password', default=None, help='the password use to login.')
@click.option('-v', '--verbose', help='sets log level to debug', is_flag=True)
def cli(host: str, port: int, verbose: bool, user: str, password: str):
if verbose:
config.console_log_level = logging.DEBUG
else:
config.console_log_level = logging.WARNING
global api_base
api_base = 'http://%s:%d/nomad/api' % (host, port)
global _cli_client
def _cli_client(): # pylint: disable=W0612
if user is not None:
return create_client(host=host, port=port, user=user, password=password)
else:
return create_client(host=host, port=port)
@cli.command(
help='Upload files to nomad. The given path can be a single file or a directory. '
'All .zip files in a directory will be uploaded.')
@click.argument('PATH', nargs=-1, required=True, type=click.Path(exists=True))
@click.option(
'--name',
help='Optional name for the upload of a single file. Will be ignored on directories.')
@click.option(
'--offline', is_flag=True, default=False,
help='Upload files "offline": files will not be uploaded, but processed were they are. '
'Only works when run on the nomad host.')
@click.option(
'--commit', is_flag=True, default=False,
help='Automatically move upload out of the staging area after successful processing')
def upload(path, name: str, offline: bool, commit: bool):
utils.configure_logging()
paths = path
click.echo('uploading files from %s paths' % len(paths))
for path in paths:
click.echo('uploading %s' % path)
if os.path.isfile(path):
name = name if name is not None else os.path.basename(path)
upload_file(path, name, offline, commit)
elif os.path.isdir(path):
for (dirpath, _, filenames) in os.walk(path):
for filename in filenames:
if filename.endswith('.zip'):
file_path = os.path.abspath(os.path.join(dirpath, filename))
name = os.path.basename(file_path)
upload_file(file_path, name, offline, commit)
else:
click.echo('Unknown path type %s.' % path)
@cli.command(help='Attempts to reset the nomad.')
def reset():
_cli_client().admin.exec_reset_command().response()
@cli.command(help='Run processing locally.')
@click.argument('ARCHIVE_ID', nargs=1, required=True, type=str)
@click.option(
......@@ -312,47 +148,3 @@ def local(archive_id, **kwargs):
backend = local.parse()
local.normalize_all(parser_backend=backend)
# backend.write_json(sys.stdout, pretty=True)
@cli.group(help='Run a nomad service locally (outside docker).')
def run():
pass
@run.command(help='Run the nomad development worker.')
def worker():
config.service = 'nomad_worker'
from nomad import processing
processing.app.worker_main(['worker', '--loglevel=INFO'])
@run.command(help='Run the nomad development api.')
@click.option('--debug', help='Does run flask in debug.', is_flag=True)
def api(debug: bool):
config.service = 'nomad_api'
from nomad import infrastructure
from nomad.api.__main__ import run_dev_server
infrastructure.setup()
run_dev_server(debug=debug, port=8000)
@cli.command(help='Runs tests and linting. Useful before commit code.')
@click.option('--skip-tests', help='Do not test, just do code checks.', is_flag=True)
def qa(skip_tests: bool):
os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
ret_code = 0
if not skip_tests:
click.echo('Run tests ...')
ret_code += os.system('python -m pytest tests')
click.echo('Run code style checks ...')
ret_code += os.system('python -m pycodestyle --ignore=E501,E701 nomad tests')
click.echo('Run linter ...')
ret_code += os.system('python -m pylint --load-plugins=pylint_mongoengine nomad tests')
click.echo('Run static type checks ...')
ret_code += os.system('python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests')
sys.exit(ret_code)
if __name__ == '__main__':
cli() # pylint: disable=E1120
nomad/client/main.py 0 → 100644
View file @ abad0d7e
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
import requests
import click
import logging
from bravado.requests_client import RequestsClient
from bravado.client import SwaggerClient
from nomad import config
api_base = 'http://%s:%d/%s' % (config.services.api_host, config.services.api_port, config.services.api_base_path)
user = 'leonard.hofstadter@nomad-fairdi.tests.de'
pw = 'password'
def create_client():
return create_client()
def _create_client(
host: str = config.services.api_host,
port: int = config.services.api_port,
base_path: str = config.services.api_base_path,
user: str = user, password: str = pw):
""" A factory method to create the client. """
http_client = RequestsClient()
if user is not None:
http_client.set_basic_auth(host, user, pw)
client = SwaggerClient.from_url(
'http://%s:%d%s/swagger.json' % (host, port, base_path),
http_client=http_client)
return client
def handle_common_errors(func):
def wrapper(*args, **kwargs):
try:
func(*args, **kwargs)
except requests.exceptions.ConnectionError:
click.echo(
'\nCould not connect to nomad at %s. '
'Check connection and host/port options.' % api_base)
sys.exit(0)
return wrapper
@click.group()
@click.option('-h', '--host', default=config.services.api_host, help='The host nomad runs on, default is "%s".' % config.services.api_host)
@click.option('-p', '--port', default=config.services.api_port, help='the port nomad runs with, default is %d.' % config.services.api_port)
@click.option('-u', '--user', default=None, help='the user name to login, default no login.')
@click.option('-w', '--password', default=None, help='the password use to login.')
@click.option('-v', '--verbose', help='sets log level to debug', is_flag=True)
def cli(host: str, port: int, verbose: bool, user: str, password: str):
if verbose:
config.console_log_level = logging.DEBUG
else:
config.console_log_level = logging.WARNING
global api_base
api_base = 'http://%s:%d/nomad/api' % (host, port)
global create_client
def create_client(): # pylint: disable=W0612
if user is not None:
return _create_client(host=host, port=port, user=user, password=password)
else:
return _create_client(host=host, port=port)
nomad/client/migration.py 0 → 100644
View file @ abad0d7e
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import click
from nomad import config, infrastructure
from nomad.migration import NomadCOEMigration
from .main import cli
_migration: NomadCOEMigration = None
@cli.group(help='Migrate data from NOMAD CoE to nomad@FAIRDI')
@click.option('-h', '--host', default=config.migration_source_db.host, help='The migration repository source db host, default is "%s".' % config.migration_source_db.host)
@click.option('-p', '--port', default=config.migration_source_db.port, help='The migration repository source db port, default is %d.' % config.migration_source_db.port)
@click.option('-u', '--user', default=config.migration_source_db.user, help='The migration repository source db user, default is %s.' % config.migration_source_db.user)
@click.option('-w', '--password', default=config.migration_source_db.password, help='The migration repository source db password.')
@click.option('-db', '--dbname', default=config.migration_source_db.dbname, help='The migration repository source db name, default is %s.' % config.migration_source_db.dbname)
def migration(host, port, user, password, dbname):
infrastructure.setup_logging()
infrastructure.setup_repository_db(
readony=True, host=host, port=port, user=user, password=password, dbname=dbname)
infrastructure.setup_mongo()
global _migration
_migration = NomadCOEMigration()
@migration.command(help='Create/update the coe repository db migration index')
@click.option('--drop', help='Drop the existing index, otherwise it will only add new data.', is_flag=True)
def index(drop):
_migration.index(drop=drop)
@migration.command(help='Copy users from source into empty target db')
@click.option('-h', '--host', default=config.repository_db.host, help='The migration repository target db host, default is "%s".' % config.repository_db.host)
@click.option('-p', '--port', default=config.repository_db.port, help='The migration repository target db port, default is %d.' % config.repository_db.port)
@click.option('-u', '--user', default=config.repository_db.user, help='The migration repository target db user, default is %s.' % config.repository_db.user)
@click.option('-w', '--password', default=config.repository_db.password, help='The migration repository target db password.')
@click.option('-db', '--dbname', default=config.repository_db.dbname, help='The migration repository target db name, default is %s.' % config.repository_db.dbname)
def copy_users(**kwargs):
_, db = infrastructure.sqlalchemy_repository_db(readonly=False, **kwargs)
_migration.copy_users(db)
@migration.command(help='Upload the given upload locations. Uses the existing index to provide user metadata.')
def upload():
pass
nomad/client/misc.py 0 → 100644
View file @ abad0d7e
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os.path
import os
import sys
import click
from nomad import config
from .main import cli, create_client
@cli.command(help='Attempts to reset the nomad.')
def reset():
create_client().admin.exec_reset_command().response()
@cli.group(help='Run a nomad service locally (outside docker).')
def run():
pass
@run.command(help='Run the nomad development worker.')
def worker():
config.service = 'nomad_worker'
from nomad import processing
processing.app.worker_main(['worker', '--loglevel=INFO'])
@run.command(help='Run the nomad development api.')
@click.option('--debug', help='Does run flask in debug.', is_flag=True)
def api(debug: bool):
config.service = 'nomad_api'
from nomad import infrastructure
from nomad.api.__main__ import run_dev_server
infrastructure.setup()
run_dev_server(debug=debug, port=8000)
@cli.command(help='Runs tests and linting. Useful before commit code.')
@click.option('--skip-tests', help='Do not test, just do code checks.', is_flag=True)
def qa(skip_tests: bool):
os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
ret_code = 0
if not skip_tests:
click.echo('Run tests ...')
ret_code += os.system('python -m pytest tests')
click.echo('Run code style checks ...')
ret_code += os.system('python -m pycodestyle --ignore=E501,E701 nomad tests')
click.echo('Run linter ...')
ret_code += os.system('python -m pylint --load-plugins=pylint_mongoengine nomad tests')
click.echo('Run static type checks ...')
ret_code += os.system('python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests')
sys.exit(ret_code)
nomad/client/upload.py 0 → 100644
View file @ abad0d7e
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os.path
import os
import time
import click
from nomad import utils
from .main import cli, create_client
def upload_file(file_path: str, name: str = None, offline: bool = False, commit: bool = False, client=None):
"""
Upload a file to nomad.
Arguments:
file_path: path to the file, absolute or relative to call directory
name: optional name, default is the file_path's basename
offline: allows to process data without upload, requires client to be run on the server
commit: automatically commit after successful processing
Returns: The upload_id
"""
if client is None:
client = create_client()
if offline:
upload = client.uploads.upload(
local_path=os.path.abspath(file_path), name=name).reponse().result
click.echo('process offline: %s' % file_path)
else:
with open(file_path, 'rb') as f:
upload = client.uploads.upload(file=f, name=name).response().result
click.echo('process online: %s' % file_path)
while upload.tasks_status not in ['SUCCESS', 'FAILURE']:
upload = client.uploads.get_upload(upload_id=upload.upload_id).response().result
calcs = upload.calcs.pagination
if calcs is None:
total, successes, failures = 0, 0, 0
else:
total, successes, failures = (calcs.total, calcs.successes, calcs.failures)
ret = '\n' if upload.tasks_status in ('SUCCESS', 'FAILURE') else '\r'
print(
'status: %s; task: %s; parsing: %d/%d/%d %s' %
(upload.tasks_status, upload.current_task, successes, failures, total, ret), end='')
time.sleep(3)
if upload.tasks_status == 'FAILURE':
click.echo('There have been errors:')
for error in upload.errors:
click.echo(' %s' % error)
elif commit:
client.uploads.exec_upload_command(upload_id=upload.upload_id, operation='commit').reponse()
return upload.upload_id
@cli.command(
help='Upload files to nomad. The given path can be a single file or a directory. '
'All .zip files in a directory will be uploaded.')
@click.argument('PATH', nargs=-1, required=True, type=click.Path(exists=True))
@click.option(
'--name',
help='Optional name for the upload of a single file. Will be ignored on directories.')
@click.option(
'--offline', is_flag=True, default=False,
help='Upload files "offline": files will not be uploaded, but processed were they are. '
'Only works when run on the nomad host.')
@click.option(
'--commit', is_flag=True, default=False,
help='Automatically move upload out of the staging area after successful processing')
def upload(path, name: str, offline: bool, commit: bool):
utils.configure_logging()
paths = path
click.echo('uploading files from %s paths' % len(paths))
for path in paths:
click.echo('uploading %s' % path)
if os.path.isfile(path):
name = name if name is not None else os.path.basename(path)
upload_file(path, name, offline, commit)
elif os.path.isdir(path):
for (dirpath, _, filenames) in os.walk(path):
for filename in filenames:
if filename.endswith('.zip'):
file_path = os.path.abspath(os.path.join(dirpath, filename))
name = os.path.basename(file_path)
upload_file(file_path, name, offline, commit)