diff --git a/docs/setup.md b/docs/setup.md
index f4a0904cb0bbea8fe0ce75b78bee72bffad14443..97adefbfbd5019c02a78a675338f220818d46835 100644
--- a/docs/setup.md
+++ b/docs/setup.md
@@ -206,7 +206,12 @@ development, like running them in a debugger, profiler, etc.
### Run the nomad worker manually
-To simply run a worker do (from the root)
+To simply run a worker with the installed nomad cli, do (from the root)
+```
+nomad run worker
+```
+
+To run it manually with celery, do (from the root)
```
celery -A nomad.processing worker -l info
```
@@ -228,6 +233,11 @@ watchmedo auto-restart -d ./nomad -p '*.py' -- celery worker -l info -A nomad.pr
### Run the api
Either with docker, or:
```
+nomad run api
+```
+
+Or manually:
+```
python nomad/api.py
```
diff --git a/nomad/api.py b/nomad/api.py
index 2e1df2349666f37c8e9746daa5c5ee84dac9ef19..20cb9241b6ace2219f05f3208f0b9182e4b67734 100644
--- a/nomad/api.py
+++ b/nomad/api.py
@@ -733,6 +733,8 @@ def get_calc(upload_hash, calc_hash):
def call_admin_operation(operation):
if operation == 'repair_uploads':
Upload.repair_all()
+ if operation == 'reset':
+ infrastructure.reset()
else:
abort(400, message='Unknown operation %s' % operation)
diff --git a/nomad/client.py b/nomad/client.py
index 2f75fc584486a5524f66d3b87b0d44bf98c1b3c3..5fc25d93205c437aa64f51b5ea4d8eb5db859cc7 100644
--- a/nomad/client.py
+++ b/nomad/client.py
@@ -27,6 +27,8 @@ from requests.auth import HTTPBasicAuth
import click
api_base = 'http://localhost/nomad/api'
+user = 'other@gmail.com'
+pw = 'nomad'
def handle_common_errors(func):
@@ -42,7 +44,7 @@ def handle_common_errors(func):
@handle_common_errors
-def upload_file(file_path, name=None, offline=False, user='other@gmail.com', pw='nomad'):
+def upload_file(file_path, name=None, offline=False):
"""
Upload a file to nomad.
@@ -143,7 +145,50 @@ def upload(path, name: str, offline: bool):
upload_file(path, name, offline)
else:
- sys.exit(0)
+ sys.exit(1)
+
+
+@cli.command(help='Attempts to reset the nomad.')
+def reset():
+ response = requests.post('%s/admin/reset' % api_base, auth=HTTPBasicAuth(user, pw))
+ if response.status_code != 200:
+ click.echo('API return %s' % str(response.status_code))
+ click.echo(response.text)
+ sys.exit(1)
+
+
+@cli.group(help='Run a nomad service locally (outside docker).')
+def run():
+ pass
+
+
+@run.command(help='Run the nomad development worker.')
+def worker():
+ from nomad import processing
+ processing.app.worker_main(['worker', '--loglevel=INFO'])
+
+
+@run.command(help='Run the nomad development api.')
+def api():
+ from nomad import infrastructure, api
+ infrastructure.setup()
+ api.app.run(debug=True, port=8000)
+
+
+@cli.command(help='Runs tests and linting. Useful before commit code.')
+def qa():
+ os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+ ret_code = 0
+ click.echo('Run tests ...')
+ ret_code += os.system('python -m pytest tests')
+ click.echo('Run code style checks ...')
+ ret_code += os.system('python -m pycodestyle --ignore=E501,E701 nomad tests')
+ click.echo('Run linter ...')
+ ret_code += os.system('python -m pylint --load-plugins=pylint_mongoengine nomad tests')
+ click.echo('Run static type checks ...')
+ ret_code += os.system('python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests')
+
+ sys.exit(ret_code)
if __name__ == '__main__':
diff --git a/nomad/dependencies.py b/nomad/dependencies.py
index 093f62fa25454b73c413ac07ca8e83956be85c78..032f32f34c42412c23d5f8468f1cf7b5f06b286b 100644
--- a/nomad/dependencies.py
+++ b/nomad/dependencies.py
@@ -43,6 +43,7 @@ import os
import os.path
import logging
import subprocess
+import shutil
_meta_info_path = './submodules/nomad-meta-info/meta_info/nomad_meta_info/'
_logger = logging.getLogger(__name__)
@@ -204,8 +205,12 @@ if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Install dependencies from NOMAD-coe.')
parser.add_argument('--dev', help='pip install with -e', action='store_true')
+ parser.add_argument('--clean', help='remove old deps first', action='store_true')
args = parser.parse_args()
_logger.setLevel(logging.DEBUG)
+ if args.clean and os.path.exists(base_dir):
+ shutil.rmtree(base_dir)
+
prepare(dev=args.dev)
diff --git a/nomad/infrastructure.py b/nomad/infrastructure.py
index ef943a6a50e1eb8ed1bd7a79f5c6150ccaf9f1ef..103424b0d60f10e514e7465b8c463a383b8866dd 100644
--- a/nomad/infrastructure.py
+++ b/nomad/infrastructure.py
@@ -17,6 +17,7 @@ This module provides function to establish connections to the database, searchen
infrastructure services.
"""
+import shutil
from mongoengine import connect
from elasticsearch_dsl import connections
from elasticsearch.exceptions import RequestError
@@ -28,6 +29,9 @@ logger = utils.get_logger(__name__)
elastic_client = None
""" The elastic search client. """
+mongo_client = None
+""" The pymongo mongodb client. """
+
def setup():
""" Creates connections to mongodb and elastic search. """
@@ -41,7 +45,8 @@ def setup():
def setup_mongo():
""" Creates connection to mongodb. """
- connect(db=config.mongo.users_db, host=config.mongo.host, port=config.mongo.port)
+ global mongo_client
+ mongo_client = connect(db=config.mongo.users_db, host=config.mongo.host, port=config.mongo.port)
logger.info('setup mongo connection')
@@ -61,3 +66,17 @@ def setup_elastic():
raise e
else:
logger.info('init elastic index')
+
+
+def reset():
+ """ Resets the databases mongo/user and elastic/calcs. Be careful. """
+ mongo_client.drop_database(config.mongo.users_db)
+ from nomad import user
+ user.ensure_test_users()
+
+ elastic_client.indices.delete(index=config.elastic.calc_index)
+ from nomad.repo import RepoCalc
+ RepoCalc.init()
+
+ shutil.rmtree(config.fs.objects, ignore_errors=True)
+ shutil.rmtree(config.fs.tmp, ignore_errors=True)
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 388efce37857a25b22a109dd6f53a22e312ca26e..ba48e9d74fccd867daf57892ad00ebdd23497a29 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -3,7 +3,7 @@ gitpython
mypy
pylint
pylint_plugin_utils
-pylint-mongoengine
+pylint_mongoengine
pycodestyle
pytest
pytest-timeout