Commit ff002332 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Fixed infrastructure dependencies of tests. Explicit connect for mongo and elastic.

parent 681c4fd4
......@@ -60,7 +60,6 @@ tests:
# fix issue with running elastic in gitlab ci runner:
# https://gitlab.com/gitlab-org/gitlab-ce/issues/42214
command: [ "bin/elasticsearch", "-Ediscovery.type=single-node" ]
- mongo:latest
variables:
RABBITMQ_ERLANG_COOKIE: SWQOKODSQALRPCLNMEQG
RABBITMQ_DEFAULT_USER: rabbitmq
......
......@@ -20,7 +20,7 @@ from elasticsearch.exceptions import NotFoundError
from datetime import datetime
import os.path
from nomad import config
from nomad import config, infrastructure
from nomad.files import UploadFile, ArchiveFile, ArchiveLogFile
from nomad.utils import get_logger
from nomad.processing import Upload, NotAllowedDuringProcessing
......@@ -747,4 +747,5 @@ api.add_resource(RepoCalcRes, '%s/repo/<string:upload_hash>/<string:calc_hash>'
if __name__ == '__main__':
infrastructure.setup()
app.run(debug=True, port=8000)
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
This module provides function to establish connections to the database, searchengine, etc.
infrastructure services.
"""
from mongoengine import connect
from elasticsearch_dsl import connections
from elasticsearch.exceptions import RequestError
from nomad import config
elastic_client = None
""" The elastic search client. """
def setup():
""" Creates connections to mongodb and elastic search. """
global elastic_client
connect(db=config.mongo.users_db, host=config.mongo.host, port=config.mongo.port)
elastic_client = connections.create_connection(hosts=[config.elastic.host])
from nomad import user
user.ensure_test_users()
def setup_mongo():
""" Creates connection to mongodb. """
connect(db=config.mongo.users_db, host=config.mongo.host, port=config.mongo.port)
def setup_elastic():
""" Creates connection to elastic search. """
global elastic_client
elastic_client = connections.create_connection(hosts=[config.elastic.host])
try:
from nomad.repo import RepoCalc
RepoCalc.init()
except RequestError as e:
if e.status_code == 400 and 'resource_already_exists_exception' in e.error:
pass # happens if two services try this at the same time
else:
raise e
......@@ -18,21 +18,16 @@ import time
from celery import Celery
from celery.signals import after_setup_task_logger, after_setup_logger, worker_process_init
from mongoengine import Document, StringField, ListField, DateTimeField, IntField, \
connect, ValidationError, BooleanField
ValidationError, BooleanField
from mongoengine.connection import MongoEngineConnectionError
from mongoengine.base.metaclasses import TopLevelDocumentMetaclass
from pymongo import ReturnDocument
from datetime import datetime
import sys
from nomad import config, utils
from nomad import config, utils, infrastructure
import nomad.patch # pylint: disable=unused-import
def mongo_connect():
return connect(db=config.mongo.users_db, host=config.mongo.host, port=config.mongo.port)
if config.logstash.enabled:
def initialize_logstash(logger=None, loglevel=logging.DEBUG, **kwargs):
utils.add_logstash_handler(logger)
......@@ -41,14 +36,10 @@ if config.logstash.enabled:
after_setup_task_logger.connect(initialize_logstash)
after_setup_logger.connect(initialize_logstash)
worker_process_init.connect(lambda **kwargs: mongo_connect())
worker_process_init.connect(lambda **kwargs: infrastructure.setup())
app = Celery('nomad.processing', broker=config.celery.broker_url)
# ensure elastic and mongo connections
if 'sphinx' not in sys.modules:
connect(db=config.mongo.users_db, host=config.mongo.host, port=config.mongo.port)
PENDING = 'PENDING'
RUNNING = 'RUNNING'
......
......@@ -24,23 +24,17 @@ is an elasticsearch_dsl document that is used to represent repository index entr
"""
from typing import Dict, Any
import sys
from elasticsearch.exceptions import ConflictError, RequestError, ConnectionTimeout
from elasticsearch_dsl import Document as ElasticDocument, Search, Date, Keyword, Boolean, \
connections
from elasticsearch.exceptions import ConflictError, ConnectionTimeout
from elasticsearch_dsl import Document as ElasticDocument, Search, Date, Keyword, Boolean
from datetime import datetime
import time
from nomad import config
from nomad import config, infrastructure
from nomad.parsing import LocalBackend
from nomad.utils import get_logger
logger = get_logger(__name__)
# ensure elastic and mongo connections
if 'sphinx' not in sys.modules:
client = connections.create_connection(hosts=[config.elastic.host])
key_mappings = {
'basis_set_type': 'program_basis_set_type',
'chemical_composition': 'chemical_composition_bulk_reduced'
......@@ -200,12 +194,12 @@ class RepoCalc(ElasticDocument):
@staticmethod
def es_search(body):
""" Perform an elasticsearch and not elasticsearch_dsl search on the Calc index. """
return client.search(index=config.elastic.calc_index, body=body)
return infrastructure.elastic_client.search(index=config.elastic.calc_index, body=body)
@staticmethod
def upload_exists(upload_hash):
""" Returns true if there are already calcs from the given upload. """
search = Search(using=client, index=config.elastic.calc_index) \
search = Search(using=infrastructure.elastic_client, index=config.elastic.calc_index) \
.query('match', upload_hash=upload_hash) \
.execute()
......@@ -223,13 +217,3 @@ class RepoCalc(ElasticDocument):
data['archive_id'] = self.archive_id
return {key: value for key, value in data.items() if value is not None}
if 'sphinx' not in sys.modules:
try:
RepoCalc.init()
except RequestError as e:
if e.status_code == 400 and 'resource_already_exists_exception' in e.error:
pass # happens if two services try this at the same time
else:
raise e
......@@ -16,7 +16,6 @@
Module with some prototypes/placeholder for future user management in nomad@FAIR.
"""
import sys
from mongoengine import Document, EmailField, StringField, ReferenceField, ListField
from passlib.apps import custom_app_context as pwd_context
from itsdangerous import TimedJSONWebSignatureSerializer as Serializer, BadSignature, SignatureExpired
......@@ -93,7 +92,3 @@ def ensure_test_users():
name='Other User')
me.hash_password('nomad')
me.save()
if 'sphinx' not in sys.modules:
ensure_test_users()
......@@ -2,7 +2,7 @@ import pytest
from mongoengine import connect
from mongoengine.connection import disconnect
from nomad import config
from nomad import config, user, infrastructure
@pytest.fixture(scope='session')
......@@ -69,32 +69,48 @@ def worker(patched_celery, celery_inspect, celery_session_worker):
@pytest.fixture(scope='function')
def mockmongo(monkeypatch):
def mock_connect(**kwargs):
return connect('test_db', host='mongomock://localhost')
disconnect()
connection = mock_connect()
monkeypatch.setattr('nomad.processing.base.mongo_connect', mock_connect)
connection = connect('test_db', host='mongomock://localhost')
monkeypatch.setattr('nomad.infrastructure.setup_mongo', lambda **kwargs: None)
user.ensure_test_users()
yield
connection.drop_database('test_db')
@pytest.fixture(scope='session')
def elastic():
infrastructure.setup_elastic()
assert infrastructure.elastic_client is not None
@pytest.fixture(scope='function')
def mocksearch(monkeypatch):
uploads = []
uploads_by_hash = {}
uploads_by_id = {}
def create_from_backend(_, **kwargs):
upload_hash = kwargs.get('upload_hash', None)
uploads.append(upload_hash)
upload_hash = kwargs['upload_hash']
upload_id = kwargs['upload_id']
uploads_by_hash[upload_hash] = (upload_id, upload_hash)
uploads_by_id[upload_id] = (upload_id, upload_hash)
return {}
def upload_exists(upload_hash):
return upload_hash in uploads
return upload_hash in uploads_by_hash
def delete_upload(upload_id):
if upload_id in uploads_by_id:
hash, id = uploads_by_id[upload_id]
del(uploads_by_id[id])
del(uploads_by_hash[hash])
monkeypatch.setattr('nomad.repo.RepoCalc.create_from_backend', create_from_backend)
monkeypatch.setattr('nomad.repo.RepoCalc.upload_exists', upload_exists)
monkeypatch.setattr('nomad.repo.RepoCalc.delete_upload', delete_upload)
monkeypatch.setattr('nomad.repo.RepoCalc.unstage', lambda *args, **kwargs: None)
@pytest.fixture(scope='function')
......
......@@ -24,10 +24,10 @@ from datetime import datetime
import shutil
import os.path
from nomad import user
from nomad.files import UploadFile, ArchiveFile, ArchiveLogFile
from nomad.processing import Upload, Calc
from nomad.processing.base import task as task_decorator
from nomad.user import me
from nomad.repo import RepoCalc
from tests.test_files import example_file, empty_file
......@@ -39,7 +39,7 @@ example_files = [empty_file, example_file]
@pytest.fixture(scope='function', autouse=True)
def mocksearch_forall(mocksearch):
def mocks_forall(mocksearch, mockmongo):
pass
......@@ -54,7 +54,7 @@ def uploaded_id(request, clear_files) -> Generator[str, None, None]:
def run_processing(uploaded_id: str) -> Upload:
upload = Upload.create(upload_id=uploaded_id, user=me)
upload = Upload.create(upload_id=uploaded_id, user=user.me)
upload.upload_time = datetime.now()
assert upload.status == 'RUNNING'
......
......@@ -144,7 +144,7 @@ def test_create_upload_with_local_path(client, test_user_auth, no_warn):
assert upload['local_path'] == 'test_local_path'
def test_delete_empty_upload(client, test_user_auth, no_warn):
def test_delete_empty_upload(client, mocksearch, test_user_auth, no_warn):
rv = client.post('/uploads', headers=test_user_auth)
assert rv.status_code == 200
......
......@@ -26,7 +26,7 @@ from tests.test_parsing import parsed_template_example # pylint: disable=unused
@pytest.fixture(scope='function')
def example_elastic_calc(normalized_template_example: LocalBackend) \
def example_elastic_calc(normalized_template_example: LocalBackend, elastic) \
-> Generator[RepoCalc, None, None]:
try:
calc = RepoCalc.get(id='test_upload_hash/test_calc_hash')
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment