Commit d6fa8c46 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Fixed various issues with mirror and re-process cli functions.

parent 431daf50
Pipeline #52478 passed with stages
in 15 minutes and 31 seconds
......@@ -493,7 +493,7 @@ class UploadResource(Resource):
return upload, 200
elif operation == 're-process':
if upload.tasks_running or not upload.published:
if upload.tasks_running or upload.process_running or not upload.published:
abort(400, message='Can only non processing, re-process published uploads')
if len(metadata) > 0:
......@@ -502,6 +502,7 @@ class UploadResource(Resource):
if len(upload.outdated_calcs) == 0:
abort(400, message='You can only re-process uploads with at least one outdated calculation')
upload.reset()
upload.re_process_upload()
return upload, 200
......
......@@ -181,6 +181,7 @@ def re_process(ctx, uploads, parallel: int):
current_task=upload.current_task, upload_id=upload.upload_id)
else:
upload.reset()
upload.re_process_upload()
upload.block_until_complete(interval=.5)
......
......@@ -15,8 +15,10 @@
import click
import sys
import json
import os
import os.path
from nomad import utils, processing as proc, search
from nomad import utils, processing as proc, search, config, files
from .client import client
......@@ -98,13 +100,18 @@ def mirror(query, move: bool, dry: bool):
search.index_all(upload.to_upload_with_metadata().calcs)
# copy/mv file
upload_files_path = upload_data.upload_files_path
if __in_test:
pass
tmp = os.path.join(config.fs.tmp, 'to_mirror')
os.rename(upload_files_path, tmp)
upload_files_path = tmp
if move:
pass
target_upload_files_path = files.PathObject(config.fs.public, upload.upload_id, create_prefix=True, prefix=True).os_path
os.rename(upload_files_path, target_upload_files_path)
os.symlink(os.path.abspath(target_upload_files_path), upload_files_path)
else:
pass
raise NotImplementedError
print(
'Mirrored %s with %d calcs at %s' %
......
......@@ -27,12 +27,13 @@ def dev():
@dev.command(help='Runs tests and linting of the nomad source code. Useful before committing code.')
@click.option('--skip-tests', help='Do no tests, just do code checks.', is_flag=True)
def qa(skip_tests: bool):
@click.option('-x', '--exitfirst', help='Stop testing after first failed test case.', is_flag=True)
def qa(skip_tests: bool, exitfirst: bool):
os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
ret_code = 0
if not skip_tests:
click.echo('Run tests ...')
ret_code += os.system('python -m pytest -svx tests')
ret_code += os.system('python -m pytest -sv%s tests' % ('x' if exitfirst else ''))
click.echo('Run code style checks ...')
ret_code += os.system('python -m pycodestyle --ignore=E501,E701 nomad tests')
click.echo('Run linter ...')
......
......@@ -509,7 +509,7 @@ def process(func):
def wrapper(self, *args, **kwargs):
assert len(args) == 0 and len(kwargs) == 0, 'process functions must not have arguments'
if self.process_running:
raise ProcessAlreadyRunning
raise ProcessAlreadyRunning('Tried to call a processing function on an already processing process.')
self.current_process = func.__name__
self.process_status = PROCESS_CALLED
......
......@@ -623,7 +623,6 @@ class Upload(Proc):
logger = self.get_logger()
logger.info('started to re-process')
self.reset()
# mock the steps of actual processing
self._continue_with('uploading')
......
......@@ -279,6 +279,7 @@ def test_re_processing(published: Upload, example_user_metadata, monkeypatch, wi
# reprocess
monkeypatch.setattr('nomad.config.version', 're_process_test_version')
monkeypatch.setattr('nomad.config.commit', 're_process_test_commit')
published.reset()
published.re_process_upload()
try:
published.block_until_complete(interval=.01)
......
......@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import pytest
import click.testing
import json
......@@ -20,11 +21,12 @@ from nomad import utils, search, processing as proc
from nomad.cli import cli
from nomad.processing import Upload, Calc
# TODO there is much more to test
@pytest.mark.usefixtures('reset_config', 'no_warn')
class TestAdmin:
def test_clean(self, published, no_warn, reset_config):
def test_clean(self, published):
upload_id = published.upload_id
Upload.objects(upload_id=upload_id).delete()
......@@ -41,9 +43,10 @@ class TestAdmin:
assert search.entry_search(search_parameters=dict(upload_id=upload_id))['pagination']['total'] > 0
@pytest.mark.usefixtures('reset_config', 'no_warn')
class TestAdminUploads:
def test_ls(self, published, no_warn, reset_config):
def test_ls(self, published):
upload_id = published.upload_id
result = click.testing.CliRunner().invoke(
......@@ -52,7 +55,7 @@ class TestAdminUploads:
assert result.exit_code == 0
assert '1 uploads selected' in result.stdout
def test_rm(self, published, no_warn, reset_config):
def test_rm(self, published):
upload_id = published.upload_id
result = click.testing.CliRunner().invoke(
......@@ -63,7 +66,7 @@ class TestAdminUploads:
assert Upload.objects(upload_id=upload_id).first() is None
assert Calc.objects(upload_id=upload_id).first() is None
def test_re_process(self, published, no_warn, monkeypatch, reset_config):
def test_re_process(self, published, monkeypatch):
monkeypatch.setattr('nomad.config.version', 'test_version')
upload_id = published.upload_id
calc = Calc.objects(upload_id=upload_id).first()
......@@ -78,9 +81,12 @@ class TestAdminUploads:
assert calc.metadata['nomad_version'] == 'test_version'
@pytest.mark.usefixtures('reset_config')
class TestClient:
def test_mirror_dry(self, published, admin_user_bravado_client):
def test_mirror_dry(self, published, admin_user_bravado_client, monkeypatch):
monkeypatch.setattr('nomad.cli.client.mirror.__in_test', True)
result = click.testing.CliRunner().invoke(
cli, ['client', 'mirror', '--dry'], catch_exceptions=False, obj=utils.POPO())
......@@ -95,7 +101,7 @@ class TestClient:
monkeypatch.setattr('nomad.cli.client.mirror.__in_test', True)
result = click.testing.CliRunner().invoke(
cli, ['client', 'mirror'], catch_exceptions=False, obj=utils.POPO())
cli, ['client', 'mirror', '--move'], catch_exceptions=False, obj=utils.POPO())
assert result.exit_code == 0
assert proc.Upload.objects(upload_id=published.upload_id).count() == 1
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment