Commit e6fb7d44 authored by Scheidgen, Markus (mscheidg)'s avatar Scheidgen, Markus (mscheidg)

Merge branch 'migration' into 'master'


See merge request !32
parents 152ee28c e39fdbc3
Pipeline #43872 passed with stages
in 21 minutes and 56 seconds
......@@ -16,5 +16,5 @@
Swagger/bravado based python client library for the API and various usefull shell commands.
from . import local, migration, misc, upload
from . import local, migration, misc, upload, integrationtests
from .main import cli, create_client
# Copyright 2018 Markus Scheidgen
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# See the License for the specific language governing permissions and
# limitations under the License.
A command that runs some example operations on a working nomad@FAIRDI installation
as a final integration test.
import time
from .main import cli
example_file = 'tests/data/proc/'
@cli.command(help='Runs a few example operations as a test.')
def integrationtests():
from .main import create_client
client = create_client()
print('upload with multiple code data')
with open(example_file, 'rb') as f:
upload = client.uploads.upload(name='integration test upload', file=f).response().result
def get_upload():
return client.uploads.get_upload(upload_id=upload.upload_id, per_page=100).response().result
print('observe the upload process to be finished')
upload = get_upload()
while upload.tasks_running:
upload = get_upload()
assert upload.tasks_status == 'SUCCESS'
total =
assert 100 > total > 0
assert len(upload.calcs.results) == total
print('get repo data')
for calc in upload.calcs.results:
repo = client.repo.get_repo_calc(upload_id=upload.upload_id, calc_id=calc.calc_id).response().result
repo['calc_id'] == calc.calc_id
print('get archive data')
for calc in upload.calcs.results:
client.archive.get_archive_calc(upload_id=upload.upload_id, calc_id=calc.calc_id).response()
print('get archive logs')
for calc in upload.calcs.results:
client.archive.get_archive_logs(upload_id=upload.upload_id, calc_id=calc.calc_id).response()
print('perform search on data')
search = client.repo.get_calcs(owner='staging', per_page=100).response().result
assert >= total
assert len(search.results) <=
print('delete the upload again')
while upload.process_running:
upload = client.uploads.get_upload(upload_id=upload.upload_id).response().result
# TODO publish upload
# TODO admin delete published upload -- this functionality does not yet exist
......@@ -16,6 +16,9 @@ import os.path
import os
import sys
import click
import asyncio
from watchgod import run_process
from concurrent.futures import ProcessPoolExecutor
from nomad import config
......@@ -35,19 +38,42 @@ def run():
@run.command(help='Run the nomad development worker.')
def worker():
config.service = 'nomad_worker'
from nomad import processing['worker', '--loglevel=INFO'])
@run.command(help='Run the nomad development api.')
@click.option('--debug', help='Does run flask in debug.', is_flag=True)
def api(debug: bool):
def run_api(**kwargs):
config.service = 'nomad_api'
from nomad import infrastructure
from nomad.api.__main__ import run_dev_server
run_dev_server(debug=debug, port=8000)
run_dev_server(port=8000, **kwargs)
def run_worker():
from nomad import processing['worker', '--loglevel=INFO'])
def run_watched_worker():
run_process('./nomad', run_worker)
def run_watched_api():
run_process('./nomad', run_api)
@run.command(help='Run both api and worker with watchdog.')
def apiworker():
executor = ProcessPoolExecutor(2)
loop = asyncio.get_event_loop()
loop.run_in_executor(executor, run_watched_worker)
loop.run_in_executor(executor, run_watched_api)
@cli.command(help='Runs tests and linting. Useful before commit code.')
......@@ -26,7 +26,7 @@ spec:
memory: "{{ .Values.worker.memlimit }}Gi"
memory: "{{ .Values.worker.memrequest }}Gi"
memory: "{{ .Values.worker.memrequest }}Gi"
- mountPath: /app/.volumes/fs
name: files-volume
python nomad/ /nomad/nomadlab/raw-data/data/ReU/
python nomad/ /nomad/nomadlab/raw-data/data/R--/
python nomad/ /nomad/nomadlab/raw-data/data/R-1/
python nomad/ /nomad/nomadlab/raw-data/data/R2V/
python nomad/ /nomad/nomadlab/raw-data/data/R6J/
\ No newline at end of file
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment