Commit e6fb7d44 authored by Markus Scheidgen's avatar Markus Scheidgen

Merge branch 'migration' into 'master'

Migration

See merge request !32
parents 152ee28c e39fdbc3
Pipeline #43872 passed with stages
in 21 minutes and 56 seconds
......@@ -16,5 +16,5 @@
Swagger/bravado based python client library for the API and various usefull shell commands.
"""
from . import local, migration, misc, upload
from . import local, migration, misc, upload, integrationtests
from .main import cli, create_client
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
A command that runs some example operations on a working nomad@FAIRDI installation
as a final integration test.
"""
import time
from .main import cli
example_file = 'tests/data/proc/examples_vasp.zip'
@cli.command(help='Runs a few example operations as a test.')
def integrationtests():
from .main import create_client
client = create_client()
print('upload with multiple code data')
with open(example_file, 'rb') as f:
upload = client.uploads.upload(name='integration test upload', file=f).response().result
def get_upload():
return client.uploads.get_upload(upload_id=upload.upload_id, per_page=100).response().result
print('observe the upload process to be finished')
upload = get_upload()
while upload.tasks_running:
time.sleep(0.3)
upload = get_upload()
assert upload.tasks_status == 'SUCCESS'
total = upload.calcs.pagination.total
assert 100 > total > 0
assert len(upload.calcs.results) == total
try:
print('get repo data')
for calc in upload.calcs.results:
repo = client.repo.get_repo_calc(upload_id=upload.upload_id, calc_id=calc.calc_id).response().result
repo['calc_id'] == calc.calc_id
print('get archive data')
for calc in upload.calcs.results:
client.archive.get_archive_calc(upload_id=upload.upload_id, calc_id=calc.calc_id).response()
print('get archive logs')
for calc in upload.calcs.results:
client.archive.get_archive_logs(upload_id=upload.upload_id, calc_id=calc.calc_id).response()
print('perform search on data')
search = client.repo.get_calcs(owner='staging', per_page=100).response().result
assert search.pagination.total >= total
assert len(search.results) <= search.pagination.total
finally:
print('delete the upload again')
client.uploads.delete_upload(upload_id=upload.upload_id)
while upload.process_running:
upload = client.uploads.get_upload(upload_id=upload.upload_id).response().result
# TODO publish upload
# TODO admin delete published upload -- this functionality does not yet exist
......@@ -16,6 +16,9 @@ import os.path
import os
import sys
import click
import asyncio
from watchgod import run_process
from concurrent.futures import ProcessPoolExecutor
from nomad import config
......@@ -35,19 +38,42 @@ def run():
@run.command(help='Run the nomad development worker.')
def worker():
config.service = 'nomad_worker'
from nomad import processing
processing.app.worker_main(['worker', '--loglevel=INFO'])
run_worker()
@run.command(help='Run the nomad development api.')
@click.option('--debug', help='Does run flask in debug.', is_flag=True)
def api(debug: bool):
run_api(debug=debug)
def run_api(**kwargs):
config.service = 'nomad_api'
from nomad import infrastructure
from nomad.api.__main__ import run_dev_server
infrastructure.setup()
run_dev_server(debug=debug, port=8000)
run_dev_server(port=8000, **kwargs)
def run_worker():
from nomad import processing
processing.app.worker_main(['worker', '--loglevel=INFO'])
def run_watched_worker():
run_process('./nomad', run_worker)
def run_watched_api():
run_process('./nomad', run_api)
@run.command(help='Run both api and worker with watchdog.')
def apiworker():
executor = ProcessPoolExecutor(2)
loop = asyncio.get_event_loop()
loop.run_in_executor(executor, run_watched_worker)
loop.run_in_executor(executor, run_watched_api)
@cli.command(help='Runs tests and linting. Useful before commit code.')
......
......@@ -26,7 +26,7 @@ spec:
limits:
memory: "{{ .Values.worker.memlimit }}Gi"
requests:
memory: "{{ .Values.worker.memrequest }}Gi"
memory: "{{ .Values.worker.memrequest }}Gi"
volumeMounts:
- mountPath: /app/.volumes/fs
name: files-volume
......
#!/bin/sh
python nomad/client.py /nomad/nomadlab/raw-data/data/ReU/ReUaDKWQVZ55N8aJqgQr48Uiljy1z.zip
python nomad/client.py /nomad/nomadlab/raw-data/data/R--/R--24XOABhczATS4I5QIDg-0MXd_8.zip
python nomad/client.py /nomad/nomadlab/raw-data/data/R-1/R-173P-ju6WxRSCnbb1eAL3gO0BtF.zip
python nomad/client.py /nomad/nomadlab/raw-data/data/R2V/R2VndW9osqfkbNZXN0ETXe8Jo8WYj.zip
python nomad/client.py /nomad/nomadlab/raw-data/data/R6J/R6JYXAnfqhWvN329Pniz0Zg6OUTIm.zip
\ No newline at end of file
/nomad/nomadlab/raw-data/data/ReU/ReUaDKWQVZ55N8aJqgQr48Uiljy1z.zip
/nomad/nomadlab/raw-data/data/R--/R--24XOABhczATS4I5QIDg-0MXd_8.zip
/nomad/nomadlab/raw-data/data/R-1/R-173P-ju6WxRSCnbb1eAL3gO0BtF.zip
/nomad/nomadlab/raw-data/data/R2V/R2VndW9osqfkbNZXN0ETXe8Jo8WYj.zip
/nomad/nomadlab/raw-data/data/R6J/R6JYXAnfqhWvN329Pniz0Zg6OUTIm.zip
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment