Commit 12d149a4 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added more utilities to cli.

parent 0938d704
Pipeline #37263 passed with stages
in 11 minutes and 43 seconds
......@@ -206,7 +206,12 @@ development, like running them in a debugger, profiler, etc.
### Run the nomad worker manually
To simply run a worker do (from the root)
To simply run a worker with the installed nomad cli, do (from the root)
```
nomad run worker
```
To run it manually with celery, do (from the root)
```
celery -A nomad.processing worker -l info
```
......@@ -228,6 +233,11 @@ watchmedo auto-restart -d ./nomad -p '*.py' -- celery worker -l info -A nomad.pr
### Run the api
Either with docker, or:
```
nomad run api
```
Or manually:
```
python nomad/api.py
```
......
......@@ -733,6 +733,8 @@ def get_calc(upload_hash, calc_hash):
def call_admin_operation(operation):
if operation == 'repair_uploads':
Upload.repair_all()
if operation == 'reset':
infrastructure.reset()
else:
abort(400, message='Unknown operation %s' % operation)
......
......@@ -27,6 +27,8 @@ from requests.auth import HTTPBasicAuth
import click
api_base = 'http://localhost/nomad/api'
user = 'other@gmail.com'
pw = 'nomad'
def handle_common_errors(func):
......@@ -42,7 +44,7 @@ def handle_common_errors(func):
@handle_common_errors
def upload_file(file_path, name=None, offline=False, user='other@gmail.com', pw='nomad'):
def upload_file(file_path, name=None, offline=False):
"""
Upload a file to nomad.
......@@ -143,7 +145,50 @@ def upload(path, name: str, offline: bool):
upload_file(path, name, offline)
else:
sys.exit(0)
sys.exit(1)
@cli.command(help='Attempts to reset the nomad.')
def reset():
response = requests.post('%s/admin/reset' % api_base, auth=HTTPBasicAuth(user, pw))
if response.status_code != 200:
click.echo('API return %s' % str(response.status_code))
click.echo(response.text)
sys.exit(1)
@cli.group(help='Run a nomad service locally (outside docker).')
def run():
pass
@run.command(help='Run the nomad development worker.')
def worker():
from nomad import processing
processing.app.worker_main(['worker', '--loglevel=INFO'])
@run.command(help='Run the nomad development api.')
def api():
from nomad import infrastructure, api
infrastructure.setup()
api.app.run(debug=True, port=8000)
@cli.command(help='Runs tests and linting. Useful before commit code.')
def qa():
os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
ret_code = 0
click.echo('Run tests ...')
ret_code += os.system('python -m pytest tests')
click.echo('Run code style checks ...')
ret_code += os.system('python -m pycodestyle --ignore=E501,E701 nomad tests')
click.echo('Run linter ...')
ret_code += os.system('python -m pylint --load-plugins=pylint_mongoengine nomad tests')
click.echo('Run static type checks ...')
ret_code += os.system('python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests')
sys.exit(ret_code)
if __name__ == '__main__':
......
......@@ -43,6 +43,7 @@ import os
import os.path
import logging
import subprocess
import shutil
_meta_info_path = './submodules/nomad-meta-info/meta_info/nomad_meta_info/'
_logger = logging.getLogger(__name__)
......@@ -204,8 +205,12 @@ if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Install dependencies from NOMAD-coe.')
parser.add_argument('--dev', help='pip install with -e', action='store_true')
parser.add_argument('--clean', help='remove old deps first', action='store_true')
args = parser.parse_args()
_logger.setLevel(logging.DEBUG)
if args.clean and os.path.exists(base_dir):
shutil.rmtree(base_dir)
prepare(dev=args.dev)
......@@ -17,6 +17,7 @@ This module provides function to establish connections to the database, searchen
infrastructure services.
"""
import shutil
from mongoengine import connect
from elasticsearch_dsl import connections
from elasticsearch.exceptions import RequestError
......@@ -28,6 +29,9 @@ logger = utils.get_logger(__name__)
elastic_client = None
""" The elastic search client. """
mongo_client = None
""" The pymongo mongodb client. """
def setup():
""" Creates connections to mongodb and elastic search. """
......@@ -41,7 +45,8 @@ def setup():
def setup_mongo():
""" Creates connection to mongodb. """
connect(db=config.mongo.users_db, host=config.mongo.host, port=config.mongo.port)
global mongo_client
mongo_client = connect(db=config.mongo.users_db, host=config.mongo.host, port=config.mongo.port)
logger.info('setup mongo connection')
......@@ -61,3 +66,17 @@ def setup_elastic():
raise e
else:
logger.info('init elastic index')
def reset():
""" Resets the databases mongo/user and elastic/calcs. Be careful. """
mongo_client.drop_database(config.mongo.users_db)
from nomad import user
user.ensure_test_users()
elastic_client.indices.delete(index=config.elastic.calc_index)
from nomad.repo import RepoCalc
RepoCalc.init()
shutil.rmtree(config.fs.objects, ignore_errors=True)
shutil.rmtree(config.fs.tmp, ignore_errors=True)
......@@ -3,7 +3,7 @@ gitpython
mypy
pylint
pylint_plugin_utils
pylint-mongoengine
pylint_mongoengine
pycodestyle
pytest
pytest-timeout
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment