Commit 7c829119 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Refactored search->repo.

parent 9be44de2
...@@ -6,7 +6,7 @@ from elasticsearch.exceptions import NotFoundError ...@@ -6,7 +6,7 @@ from elasticsearch.exceptions import NotFoundError
from nomad import config, files from nomad import config, files
from nomad.utils import get_logger, create_uuid from nomad.utils import get_logger, create_uuid
from nomad.processing import Upload, Calc, NotAllowedDuringProcessing from nomad.processing import Upload, Calc, NotAllowedDuringProcessing
from nomad.search import CalcElasticDocument from nomad.repo import RepoCalc
from nomad.user import me from nomad.user import me
base_path = config.services.api_base_path base_path = config.services.api_base_path
...@@ -295,7 +295,7 @@ class RepoCalcRes(Resource): ...@@ -295,7 +295,7 @@ class RepoCalcRes(Resource):
:returns: the repository calculation entry :returns: the repository calculation entry
""" """
try: try:
return CalcElasticDocument.get(id='%s/%s' % (upload_hash, calc_hash)).json_dict, 200 return RepoCalc.get(id='%s/%s' % (upload_hash, calc_hash)).json_dict, 200
except NotFoundError: except NotFoundError:
abort(404, message='There is no calculation for %s/%s' % (upload_hash, calc_hash)) abort(404, message='There is no calculation for %s/%s' % (upload_hash, calc_hash))
except Exception as e: except Exception as e:
...@@ -372,7 +372,7 @@ class RepoCalcsRes(Resource): ...@@ -372,7 +372,7 @@ class RepoCalcsRes(Resource):
assert per_page > 0 assert per_page > 0
try: try:
search = CalcElasticDocument.search().query('match_all') search = RepoCalc.search().query('match_all')
search = search[(page - 1) * per_page: page * per_page] search = search[(page - 1) * per_page: page * per_page]
return { return {
'pagination': { 'pagination': {
......
...@@ -30,7 +30,7 @@ This module is structures into our *celery app* (``app.py``), the task definitio ...@@ -30,7 +30,7 @@ This module is structures into our *celery app* (``app.py``), the task definitio
based on file storage notifications (``handler.py``, ``handlerdaemon.py``). based on file storage notifications (``handler.py``, ``handlerdaemon.py``).
This module does not contain the functions to do the actual work. Those are encapsulated This module does not contain the functions to do the actual work. Those are encapsulated
in :py:mod:`nomad.files`, :py:mod:`nomad.search`, :py:mod:`nomad.users`, in :py:mod:`nomad.files`, :py:mod:`nomad.repo`, :py:mod:`nomad.users`,
:py:mod:`nomad.parsing`, and :py:mod:`nomad.normalizing`. :py:mod:`nomad.parsing`, and :py:mod:`nomad.normalizing`.
Processing app Processing app
......
...@@ -37,7 +37,7 @@ import mongoengine.errors ...@@ -37,7 +37,7 @@ import mongoengine.errors
import logging import logging
from nomad import config, files, utils from nomad import config, files, utils
from nomad.search import CalcElasticDocument from nomad.repo import RepoCalc
from nomad.user import User, me from nomad.user import User, me
from nomad.processing.base import Proc, process, task, PENDING from nomad.processing.base import Proc, process, task, PENDING
from nomad.parsing import LocalBackend, parsers, parser_dict from nomad.parsing import LocalBackend, parsers, parser_dict
...@@ -96,7 +96,7 @@ class Calc(Proc): ...@@ -96,7 +96,7 @@ class Calc(Proc):
files.delete_archive(self.archive_id) files.delete_archive(self.archive_id)
# delete the search index entry # delete the search index entry
elastic_entry = CalcElasticDocument.get(self.archive_id) elastic_entry = RepoCalc.get(self.archive_id)
if elastic_entry is not None: if elastic_entry is not None:
elastic_entry.delete() elastic_entry.delete()
...@@ -159,7 +159,7 @@ class Calc(Proc): ...@@ -159,7 +159,7 @@ class Calc(Proc):
def archiving(self): def archiving(self):
upload_hash, calc_hash = self.archive_id.split('/') upload_hash, calc_hash = self.archive_id.split('/')
# persist to elastic search # persist to elastic search
CalcElasticDocument.create_from_backend( RepoCalc.create_from_backend(
self._parser_backend, self._parser_backend,
upload_hash=upload_hash, upload_hash=upload_hash,
calc_hash=calc_hash, calc_hash=calc_hash,
...@@ -314,7 +314,7 @@ class Upload(Proc): ...@@ -314,7 +314,7 @@ class Upload(Proc):
self.fail('could not create upload hash', e) self.fail('could not create upload hash', e)
return return
if CalcElasticDocument.upload_exists(self.upload_hash): if RepoCalc.upload_exists(self.upload_hash):
self.fail('The same file was already uploaded and processed.', level=logging.INFO) self.fail('The same file was already uploaded and processed.', level=logging.INFO)
return return
......
...@@ -36,7 +36,7 @@ key_mappings = { ...@@ -36,7 +36,7 @@ key_mappings = {
class AlreadyExists(Exception): pass class AlreadyExists(Exception): pass
class CalcElasticDocument(ElasticDocument): class RepoCalc(ElasticDocument):
""" """
Elastic search document that represents a calculation. It is supposed to be a Elastic search document that represents a calculation. It is supposed to be a
component of :class:`Calc`. Should only be created by its parent :class:`Calc` component of :class:`Calc`. Should only be created by its parent :class:`Calc`
...@@ -72,7 +72,7 @@ class CalcElasticDocument(ElasticDocument): ...@@ -72,7 +72,7 @@ class CalcElasticDocument(ElasticDocument):
@classmethod @classmethod
def create_from_backend( def create_from_backend(
cls, backend: LocalBackend, upload_id: str, upload_hash: str, calc_hash: str, cls, backend: LocalBackend, upload_id: str, upload_hash: str, calc_hash: str,
**kwargs) -> 'CalcElasticDocument': **kwargs) -> 'RepoCalc':
""" """
Create a new calculation instance in elastic search. The data from the given backend Create a new calculation instance in elastic search. The data from the given backend
will be used. Additional meta-data can be given as *kwargs*. ``upload_id``, will be used. Additional meta-data can be given as *kwargs*. ``upload_id``,
...@@ -151,7 +151,7 @@ class CalcElasticDocument(ElasticDocument): ...@@ -151,7 +151,7 @@ class CalcElasticDocument(ElasticDocument):
if 'sphinx' not in sys.modules: if 'sphinx' not in sys.modules:
try: try:
CalcElasticDocument.init() RepoCalc.init()
except elasticsearch.exceptions.RequestError as e: except elasticsearch.exceptions.RequestError as e:
if e.status_code == 400 and 'resource_already_exists_exception' in e.error: if e.status_code == 400 and 'resource_already_exists_exception' in e.error:
pass # happens if two services try this at the same time pass # happens if two services try this at the same time
......
...@@ -41,5 +41,5 @@ def mocksearch(monkeypatch): ...@@ -41,5 +41,5 @@ def mocksearch(monkeypatch):
def upload_exists(upload_hash): def upload_exists(upload_hash):
return upload_hash in uploads return upload_hash in uploads
monkeypatch.setattr('nomad.search.CalcElasticDocument.create_from_backend', create_from_backend) monkeypatch.setattr('nomad.repo.RepoCalc.create_from_backend', create_from_backend)
monkeypatch.setattr('nomad.search.CalcElasticDocument.upload_exists', upload_exists) monkeypatch.setattr('nomad.repo.RepoCalc.upload_exists', upload_exists)
...@@ -27,7 +27,7 @@ from nomad import config, files ...@@ -27,7 +27,7 @@ from nomad import config, files
from nomad.processing import Upload, Calc from nomad.processing import Upload, Calc
from nomad.processing.base import task as task_decorator from nomad.processing.base import task as task_decorator
from nomad.user import me from nomad.user import me
from nomad.search import CalcElasticDocument from nomad.repo import RepoCalc
from tests.test_files import example_file, empty_file from tests.test_files import example_file, empty_file
...@@ -90,7 +90,7 @@ def test_processing_doublets(uploaded_id, celery_session_worker, caplog): ...@@ -90,7 +90,7 @@ def test_processing_doublets(uploaded_id, celery_session_worker, caplog):
upload = run_processing(uploaded_id) upload = run_processing(uploaded_id)
assert upload.status == 'SUCCESS' assert upload.status == 'SUCCESS'
assert CalcElasticDocument.upload_exists(upload.upload_hash) # pylint: disable=E1101 assert RepoCalc.upload_exists(upload.upload_hash) # pylint: disable=E1101
upload = run_processing(uploaded_id) upload = run_processing(uploaded_id)
assert upload.status == 'FAILURE' assert upload.status == 'FAILURE'
......
...@@ -24,7 +24,7 @@ from tests.test_files import assert_exists # noqa ...@@ -24,7 +24,7 @@ from tests.test_files import assert_exists # noqa
from tests.test_files import clear_files, archive_id # noqa pylint: disable=unused-import from tests.test_files import clear_files, archive_id # noqa pylint: disable=unused-import
from tests.test_normalizing import normalized_vasp_example # noqa pylint: disable=unused-import from tests.test_normalizing import normalized_vasp_example # noqa pylint: disable=unused-import
from tests.test_parsing import parsed_vasp_example # noqa pylint: disable=unused-import from tests.test_parsing import parsed_vasp_example # noqa pylint: disable=unused-import
from tests.test_search import example_elastic_calc # noqa pylint: disable=unused-import from tests.test_repo import example_elastic_calc # noqa pylint: disable=unused-import
@pytest.fixture(scope='function') @pytest.fixture(scope='function')
......
...@@ -21,7 +21,7 @@ from elasticsearch import NotFoundError ...@@ -21,7 +21,7 @@ from elasticsearch import NotFoundError
from nomad import config from nomad import config
from nomad.parsing import LocalBackend from nomad.parsing import LocalBackend
from nomad.search import AlreadyExists, CalcElasticDocument, key_mappings from nomad.repo import AlreadyExists, RepoCalc, key_mappings
from tests.test_normalizing import normalized_vasp_example # pylint: disable=unused-import from tests.test_normalizing import normalized_vasp_example # pylint: disable=unused-import
from tests.test_parsing import parsed_vasp_example # pylint: disable=unused-import from tests.test_parsing import parsed_vasp_example # pylint: disable=unused-import
...@@ -30,16 +30,16 @@ from tests.test_files import assert_not_exists ...@@ -30,16 +30,16 @@ from tests.test_files import assert_not_exists
@pytest.fixture(scope='function') @pytest.fixture(scope='function')
def example_elastic_calc(normalized_vasp_example: LocalBackend, caplog) \ def example_elastic_calc(normalized_vasp_example: LocalBackend, caplog) \
-> Generator[CalcElasticDocument, None, None]: -> Generator[RepoCalc, None, None]:
try: try:
caplog.set_level(logging.ERROR) caplog.set_level(logging.ERROR)
CalcElasticDocument.get(id='test_upload_hash/test_calc_hash').delete() RepoCalc.get(id='test_upload_hash/test_calc_hash').delete()
except Exception: except Exception:
pass pass
finally: finally:
caplog.set_level(logging.WARNING) caplog.set_level(logging.WARNING)
entry = CalcElasticDocument.create_from_backend( entry = RepoCalc.create_from_backend(
normalized_vasp_example, normalized_vasp_example,
upload_hash='test_upload_hash', upload_hash='test_upload_hash',
calc_hash='test_calc_hash', calc_hash='test_calc_hash',
...@@ -59,27 +59,27 @@ def example_elastic_calc(normalized_vasp_example: LocalBackend, caplog) \ ...@@ -59,27 +59,27 @@ def example_elastic_calc(normalized_vasp_example: LocalBackend, caplog) \
caplog.set_level(logging.WARNING) caplog.set_level(logging.WARNING)
def assert_elastic_calc(calc: CalcElasticDocument): def assert_elastic_calc(calc: RepoCalc):
assert calc is not None assert calc is not None
for property in CalcElasticDocument._doc_type.mapping: for property in RepoCalc._doc_type.mapping:
property = key_mappings.get(property, property) property = key_mappings.get(property, property)
assert getattr(calc, property) is not None assert getattr(calc, property) is not None
def test_create_elasitc_calc(example_elastic_calc: CalcElasticDocument): def test_create_elasitc_calc(example_elastic_calc: RepoCalc):
assert_elastic_calc(example_elastic_calc) assert_elastic_calc(example_elastic_calc)
assert CalcElasticDocument.upload_exists(example_elastic_calc.upload_hash) assert RepoCalc.upload_exists(example_elastic_calc.upload_hash)
get_result: CalcElasticDocument = CalcElasticDocument.get( get_result: RepoCalc = RepoCalc.get(
id='%s/%s' % (example_elastic_calc.upload_hash, example_elastic_calc.calc_hash)) id='%s/%s' % (example_elastic_calc.upload_hash, example_elastic_calc.calc_hash))
assert_elastic_calc(get_result) assert_elastic_calc(get_result)
def test_create_existing_elastic_calc( def test_create_existing_elastic_calc(
example_elastic_calc: CalcElasticDocument, normalized_vasp_example, caplog): example_elastic_calc: RepoCalc, normalized_vasp_example, caplog):
try: try:
caplog.set_level(logging.ERROR) caplog.set_level(logging.ERROR)
CalcElasticDocument.create_from_backend( RepoCalc.create_from_backend(
normalized_vasp_example, normalized_vasp_example,
upload_hash='test_upload_hash', upload_hash='test_upload_hash',
calc_hash='test_calc_hash', calc_hash='test_calc_hash',
...@@ -94,13 +94,13 @@ def test_create_existing_elastic_calc( ...@@ -94,13 +94,13 @@ def test_create_existing_elastic_calc(
assert False assert False
def test_delete_elastic_calc(example_elastic_calc: CalcElasticDocument, caplog): def test_delete_elastic_calc(example_elastic_calc: RepoCalc, caplog):
example_elastic_calc.delete() example_elastic_calc.delete()
assert_not_exists(config.files.archive_bucket, 'test_upload_hash/test_calc_hash') assert_not_exists(config.files.archive_bucket, 'test_upload_hash/test_calc_hash')
try: try:
caplog.set_level(logging.ERROR) caplog.set_level(logging.ERROR)
CalcElasticDocument.get(id='test_upload_hash/test_calc_hash') RepoCalc.get(id='test_upload_hash/test_calc_hash')
assert False assert False
except NotFoundError: except NotFoundError:
pass pass
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment