test_processing.py 3.29 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
16
17
18
19
20
"""
To run this test, a celery worker must be running. The test worker provided by
the celery pytest plugin is currently not working. It results on a timeout when
reading from the redis result backend, even though all task apperently ended successfully.
"""

21
22
from typing import Generator
import pytest
23
24

import nomad.config as config
25
import nomad.files as files
26
from nomad.processing import UploadProcessing
27

28
from tests.test_files import example_file, empty_file
29
30
31
# import fixtures
from tests.test_search import index  # pylint: disable=unused-import
from tests.test_files import clear_files  # pylint: disable=unused-import
32

33
example_files = [empty_file, example_file]
34
35


36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
# disable test worker for now, see docstring above
# all further celery_* fixtures become effectivly mute.
@pytest.fixture(scope='session')
def celery_session_worker():
    return None


@pytest.fixture(scope='session')
def celery_includes():
    return ['nomad.processing']


@pytest.fixture(scope='session')
def celery_config():
    return {
        'broker_url': config.celery.broker_url,
        'result_backend': config.celery.backend_url,
        'accept_content': ['json', 'pickle'],
        'task_serializer': config.celery.serializer,
        'result_serializer': config.celery.serializer
    }


Markus Scheidgen's avatar
Markus Scheidgen committed
59
@pytest.fixture(scope='function', params=example_files)
60
def uploaded_id(request, clear_files) -> Generator[str, None, None]:
Markus Scheidgen's avatar
Markus Scheidgen committed
61
62
    example_file = request.param
    example_upload_id = example_file.replace('.zip', '')
63
    files._client.fput_object(config.files.uploads_bucket, example_upload_id, example_file)
64

65
    yield example_upload_id
66

67

68
def test_processing(uploaded_id, celery_session_worker):
Markus Scheidgen's avatar
Markus Scheidgen committed
69
70
    run = UploadProcessing(uploaded_id)
    run.start()
71

72
73
74
75
76
    # test that the instance can be reinstantiated from a persistable representation
    run = UploadProcessing.from_result_backend(uploaded_id, run.result_tuple)

    assert run.status in ['PENDING', 'PROGRESS']

77
    run.get(timeout=10)
78

79
    assert run.ready()
Markus Scheidgen's avatar
Markus Scheidgen committed
80
    assert run.task_name == 'nomad.processing.close_upload'
81
    assert run.upload_hash is not None
Markus Scheidgen's avatar
Markus Scheidgen committed
82
83
    assert run.cause is None
    assert run.status == 'SUCCESS'
84
85
86
87
    for processing_result in run.processing_results:
        for status, errors in processing_result:
            assert errors is None or len(errors) == 0
            assert status in ['ParseSuccess', 'NormalizeSuccess', 'IndexSuccess', 'PersistenceSuccess']
88
89

    run.forget()
Markus Scheidgen's avatar
Markus Scheidgen committed
90
91


92
def test_process_non_existing(celery_session_worker):
Markus Scheidgen's avatar
Markus Scheidgen committed
93
94
    run = UploadProcessing('__does_not_exist')
    run.start()
95
96
97

    run.get(timeout=10)

Markus Scheidgen's avatar
Markus Scheidgen committed
98
99
100
101
102
103
104
    assert run.ready()
    run.forget()

    assert run.task_name == 'nomad.processing.open_upload'
    assert run.status == 'SUCCESS'
    assert run.cause is not None
    assert isinstance(run.cause, KeyError)