Merge branch 'migration' into 'master'

Migration

See merge request !32

nomad/client/__init__.py

@@ -16,5 +16,5 @@
 Swagger/bravado based python client library for the API and various usefull shell commands.
 """
 
-from . import local, migration, misc, upload
+from . import local, migration, misc, upload, integrationtests
 from .main import cli, create_client

nomad/client/integrationtests.py

+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+A command that runs some example operations on a working nomad@FAIRDI installation
+as a final integration test.
+"""
+
+import time
+
+from .main import cli
+
+
+example_file = 'tests/data/proc/examples_vasp.zip'
+
+
+@cli.command(help='Runs a few example operations as a test.')
+def integrationtests():
+    from .main import create_client
+    client = create_client()
+
+    print('upload with multiple code data')
+    with open(example_file, 'rb') as f:
+        upload = client.uploads.upload(name='integration test upload', file=f).response().result
+
+    def get_upload():
+        return client.uploads.get_upload(upload_id=upload.upload_id, per_page=100).response().result
+
+    print('observe the upload process to be finished')
+    upload = get_upload()
+    while upload.tasks_running:
+        time.sleep(0.3)
+        upload = get_upload()
+
+    assert upload.tasks_status == 'SUCCESS'
+    total = upload.calcs.pagination.total
+    assert 100 > total > 0
+    assert len(upload.calcs.results) == total
+
+    try:
+        print('get repo data')
+        for calc in upload.calcs.results:
+            repo = client.repo.get_repo_calc(upload_id=upload.upload_id, calc_id=calc.calc_id).response().result
+            repo['calc_id'] == calc.calc_id
+
+        print('get archive data')
+        for calc in upload.calcs.results:
+            client.archive.get_archive_calc(upload_id=upload.upload_id, calc_id=calc.calc_id).response()
+
+        print('get archive logs')
+        for calc in upload.calcs.results:
+            client.archive.get_archive_logs(upload_id=upload.upload_id, calc_id=calc.calc_id).response()
+
+        print('perform search on data')
+        search = client.repo.get_calcs(owner='staging', per_page=100).response().result
+        assert search.pagination.total >= total
+        assert len(search.results) <= search.pagination.total
+    finally:
+        print('delete the upload again')
+        client.uploads.delete_upload(upload_id=upload.upload_id)
+        while upload.process_running:
+            upload = client.uploads.get_upload(upload_id=upload.upload_id).response().result
+
+    # TODO publish upload
+    # TODO admin delete published upload -- this functionality does not yet exist

nomad/client/misc.py

@@ -16,6 +16,9 @@ import os.path
 import os
 import sys
 import click
+import asyncio
+from watchgod import run_process
+from concurrent.futures import ProcessPoolExecutor
 
 from nomad import config
 
@@ -35,19 +38,42 @@ def run():
 
 @run.command(help='Run the nomad development worker.')
 def worker():
-    config.service = 'nomad_worker'
-    from nomad import processing
-    processing.app.worker_main(['worker', '--loglevel=INFO'])
+    run_worker()
 
 
 @run.command(help='Run the nomad development api.')
 @click.option('--debug', help='Does run flask in debug.', is_flag=True)
 def api(debug: bool):
+    run_api(debug=debug)
+
+
+def run_api(**kwargs):
     config.service = 'nomad_api'
     from nomad import infrastructure
     from nomad.api.__main__ import run_dev_server
     infrastructure.setup()
-    run_dev_server(debug=debug, port=8000)
+    run_dev_server(port=8000, **kwargs)
+
+
+def run_worker():
+    from nomad import processing
+    processing.app.worker_main(['worker', '--loglevel=INFO'])
+
+
+def run_watched_worker():
+    run_process('./nomad', run_worker)
+
+
+def run_watched_api():
+    run_process('./nomad', run_api)
+
+
+@run.command(help='Run both api and worker with watchdog.')
+def apiworker():
+    executor = ProcessPoolExecutor(2)
+    loop = asyncio.get_event_loop()
+    loop.run_in_executor(executor, run_watched_worker)
+    loop.run_in_executor(executor, run_watched_api)
 
 
 @cli.command(help='Runs tests and linting. Useful before commit code.')

ops/helm/nomad/templates/worker-deployment.yaml

@@ -26,7 +26,7 @@ spec:
           limits:
             memory: "{{ .Values.worker.memlimit }}Gi"
           requests:
-            memory: "{{ .Values.worker.memrequest }}Gi"
+            memory: "{{ .Values.worker.memrequest }}Gi"
         volumeMounts:
         - mountPath: /app/.volumes/fs
           name: files-volume

requirements.txt

@@ -42,13 +42,13 @@ bravado
 PyJWT
 jsonschema[format]
 python-magic
+watchgod
 
 # dev/ops related
 setuptools
 sphinx
 sphinxcontrib.httpdomain
 sphinx_rtd_theme
-watchdog
 gitpython
 mypy
 pylint

tests/data/test_coe_uploads.sh

-#!/bin/sh
-python nomad/client.py /nomad/nomadlab/raw-data/data/ReU/ReUaDKWQVZ55N8aJqgQr48Uiljy1z.zip
-python nomad/client.py /nomad/nomadlab/raw-data/data/R--/R--24XOABhczATS4I5QIDg-0MXd_8.zip
-python nomad/client.py /nomad/nomadlab/raw-data/data/R-1/R-173P-ju6WxRSCnbb1eAL3gO0BtF.zip
-python nomad/client.py /nomad/nomadlab/raw-data/data/R2V/R2VndW9osqfkbNZXN0ETXe8Jo8WYj.zip
-python nomad/client.py /nomad/nomadlab/raw-data/data/R6J/R6JYXAnfqhWvN329Pniz0Zg6OUTIm.zip
\ No newline at end of file

tests/data/test_coe_uploads.txt

+/nomad/nomadlab/raw-data/data/ReU/ReUaDKWQVZ55N8aJqgQr48Uiljy1z.zip
+/nomad/nomadlab/raw-data/data/R--/R--24XOABhczATS4I5QIDg-0MXd_8.zip
+/nomad/nomadlab/raw-data/data/R-1/R-173P-ju6WxRSCnbb1eAL3gO0BtF.zip
+/nomad/nomadlab/raw-data/data/R2V/R2VndW9osqfkbNZXN0ETXe8Jo8WYj.zip
+/nomad/nomadlab/raw-data/data/R6J/R6JYXAnfqhWvN329Pniz0Zg6OUTIm.zip
\ No newline at end of file