test_api.py 6.46 KB
Newer Older
1
2
3
4
5
6
7
8
import pytest
from threading import Thread
import subprocess
import shlex
import time
import json
from mongoengine import connect
from mongoengine.connection import disconnect
9
from datetime import datetime, timedelta
10

11
12
13
14
15
16
17
18
19
20
from nomad import config
# for convinience we test the api without path prefix
services_config = config.services._asdict()
services_config.update(api_base_path='')
config.services = config.NomadServicesConfig(**services_config)

from nomad import api, files, processing, users  # noqa

from tests.test_processing import example_files  # noqa
from tests.test_files import assert_exists  # noqa
21

22
# import fixtures
23
24
25
26
from tests.test_files import clear_files, archive_id  # noqa pylint: disable=unused-import
from tests.test_normalizing import normalized_vasp_example  # noqa pylint: disable=unused-import
from tests.test_parsing import parsed_vasp_example  # noqa pylint: disable=unused-import
from tests.test_search import example_entry  # noqa pylint: disable=unused-import
27
from tests.test_processing import celery_config, celery_includes, mocksearch  # noqa pylint: disable=unused-import
28

29

30
@pytest.fixture(scope='function')
31
32
33
34
35
36
37
38
def client():
    disconnect()
    connect('users_test', host=config.mongo.host, is_mock=True)

    api.app.config['TESTING'] = True
    client = api.app.test_client()

    yield client
39
    users.Upload._get_collection().drop()
40
41
42
43
44
45
46
47
48
49
50


def assert_uploads(upload_json_str, count=0, **kwargs):
    data = json.loads(upload_json_str)
    assert isinstance(data, list)
    assert len(data) == count

    if count > 0:
        assert_upload(json.dumps(data[0]), **kwargs)


51
def assert_upload(upload_json_str, id=None, **kwargs):
52
    data = json.loads(upload_json_str)
Markus Scheidgen's avatar
Markus Scheidgen committed
53
    assert 'upload_id' in data
54
    if id is not None:
Markus Scheidgen's avatar
Markus Scheidgen committed
55
        assert id == data['upload_id']
56
57
58
    assert 'create_time' in data
    assert 'presigned_url' in data

59
60
61
    for key, value in kwargs.items():
        assert data.get(key, None) == value

62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
    return data


def test_no_uploads(client):
    rv = client.get('/uploads')

    assert rv.status_code == 200
    assert_uploads(rv.data, count=0)


def test_bad_upload_id(client):
    rv = client.get('/uploads/bad_id')
    assert rv.status_code == 400


def test_not_existing_upload(client):
    rv = client.get('/uploads/123456789012123456789012')
    assert rv.status_code == 404


82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
def test_stale_upload(client):
    rv = client.post(
        '/uploads',
        data=json.dumps(dict(name='test_name')),
        content_type='application/json')
    assert rv.status_code == 200
    upload_id = assert_upload(rv.data)['upload_id']

    upload = users.Upload.objects(id=upload_id).first()
    upload.create_time = datetime.now() - timedelta(days=2)
    upload.save()

    rv = client.get('/uploads/%s' % upload_id)
    assert rv.status_code == 200
    assert_upload(rv.data, is_stale=True)


99
100
101
102
def test_create_upload(client):
    rv = client.post('/uploads')

    assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
103
    upload_id = assert_upload(rv.data)['upload_id']
104
105
106

    rv = client.get('/uploads/%s' % upload_id)
    assert rv.status_code == 200
107
    assert_upload(rv.data, id=upload_id, is_stale=False)
108
109
110
111
112
113

    rv = client.get('/uploads')
    assert rv.status_code == 200
    assert_uploads(rv.data, count=1, id=upload_id)


114
115
116
117
118
119
120
121
def test_create_upload_with_name(client):
    rv = client.post(
        '/uploads', data=json.dumps(dict(name='test_name')), content_type='application/json')
    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    assert upload['name'] == 'test_name'


122
@pytest.mark.parametrize("file", example_files)
Markus Scheidgen's avatar
Markus Scheidgen committed
123
@pytest.mark.timeout(30)
124
def test_upload_to_upload(client, file):
125
126
127
128
129
130
    rv = client.post('/uploads')
    assert rv.status_code == 200
    upload = assert_upload(rv.data)

    @files.upload_put_handler
    def handle_upload_put(received_upload_id: str):
Markus Scheidgen's avatar
Markus Scheidgen committed
131
        assert upload['upload_id'] == received_upload_id
132
133
134
135
136
137
138
139
140
141
        raise StopIteration

    def handle_uploads():
        handle_upload_put(received_upload_id='provided by decorator')

    handle_uploads_thread = Thread(target=handle_uploads)
    handle_uploads_thread.start()

    time.sleep(1)
    upload_url = upload['presigned_url']
142
    cmd = files.create_curl_upload_cmd(upload_url).replace('<ZIPFILE>', file)
143
144
145
146
    subprocess.call(shlex.split(cmd))

    handle_uploads_thread.join()

Markus Scheidgen's avatar
Markus Scheidgen committed
147
    assert_exists(config.files.uploads_bucket, upload['upload_id'])
148
149
150


@pytest.mark.parametrize("file", example_files)
Markus Scheidgen's avatar
Markus Scheidgen committed
151
@pytest.mark.timeout(30)
152
def test_processing(client, file, celery_session_worker, mocksearch):
Markus Scheidgen's avatar
Markus Scheidgen committed
153
    handle_uploads_thread = processing.handle_uploads_thread(quit=True)
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168

    rv = client.post('/uploads')
    assert rv.status_code == 200
    upload = assert_upload(rv.data)

    time.sleep(1)
    upload_url = upload['presigned_url']
    cmd = files.create_curl_upload_cmd(upload_url).replace('<ZIPFILE>', file)
    subprocess.call(shlex.split(cmd))

    handle_uploads_thread.join()

    while True:
        time.sleep(1)

Markus Scheidgen's avatar
Markus Scheidgen committed
169
        rv = client.get('/uploads/%s' % upload['upload_id'])
170
171
172
        assert rv.status_code == 200
        upload = assert_upload(rv.data)
        assert 'upload_time' in upload
Markus Scheidgen's avatar
Markus Scheidgen committed
173
174
        if upload['proc']['status'] in ['SUCCESS', 'FAILURE']:
            break
175

Markus Scheidgen's avatar
Markus Scheidgen committed
176
    proc = upload['proc']
177
    assert proc['status'] == 'SUCCESS'
Markus Scheidgen's avatar
Markus Scheidgen committed
178
179
180
    assert 'calc_procs' in proc
    assert proc['calc_procs'] is not None
    assert proc['current_task_name'] == 'cleanup'
Markus Scheidgen's avatar
Markus Scheidgen committed
181
    assert len(proc['task_names']) == 4
Markus Scheidgen's avatar
Markus Scheidgen committed
182
    assert_exists(config.files.uploads_bucket, upload['upload_id'])
183
184


185
def test_repo_calc(client, example_entry):
186
187
188
189
    rv = client.get('/repo/%s/%s' % (example_entry.upload_hash, example_entry.calc_hash))
    assert rv.status_code == 200


190
def test_non_existing_repo_cals(client):
191
192
193
194
    rv = client.get('/repo/doesnt/exist')
    assert rv.status_code == 404


195
def test_repo_calcs(client, example_entry):
196
197
198
199
200
201
202
203
204
    rv = client.get('/repo')
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert isinstance(results, list)
    assert len(results) >= 1


205
206
207
208
209
210
211
212
213
214
def test_repo_calcs_pagination(client, example_entry):
    rv = client.get('/repo?page=1&per_page=1')
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert isinstance(results, list)
    assert len(results) == 1


215
216
def test_get_archive(client, archive_id):
    rv = client.get('/archive/%s' % archive_id)
217
    assert rv.status_code == 302
218
219
220
221
222


def test_get_non_existing_archive(client):
    rv = client.get('/archive/%s' % 'doesnt/exist')
    assert rv.status_code == 404